chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
Executable
BIN
Binary file not shown.
|
After Width: | Height: | Size: 8.1 KiB |
Executable
+33
@@ -0,0 +1,33 @@
|
||||
**代码目录**
|
||||
|
||||
第1章 统计学习方法概论
|
||||
|
||||
第2章 感知机
|
||||
|
||||
第3章 k近邻法
|
||||
|
||||
第4章 朴素贝叶斯
|
||||
|
||||
第5章 决策树
|
||||
|
||||
第6章 逻辑斯谛回归
|
||||
|
||||
第7章 支持向量机
|
||||
|
||||
第8章 提升方法
|
||||
|
||||
第9章 EM算法及其推广
|
||||
|
||||
第10章 隐马尔可夫模型
|
||||
|
||||
第11章 条件随机场
|
||||
|
||||
-----------
|
||||
参考：
|
||||
https://github.com/wzyonggege/statistical-learning-method
|
||||
https://github.com/WenDesi/lihang_book_algorithm
|
||||
https://blog.csdn.net/tudaodiaozhale
|
||||
|
||||
代码整理和修改：机器学习初学者 （微信公众号，ID:ai-start-com）
|
||||
|
||||
|
||||
+310
@@ -0,0 +1,310 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"原文代码作者：https://blog.csdn.net/tudaodiaozhale\n",
|
||||
"\n",
|
||||
"中文注释制作：机器学习初学者(微信公众号：ID:ai-start-com)\n",
|
||||
"\n",
|
||||
"配置环境：python 3.6\n",
|
||||
"\n",
|
||||
"代码全部测试通过。\n",
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 第10章 隐马尔可夫模型"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class HiddenMarkov:\n",
|
||||
" def forward(self, Q, V, A, B, O, PI): # 使用前向算法\n",
|
||||
" N = len(Q) # 状态序列的大小\n",
|
||||
" M = len(O) # 观测序列的大小\n",
|
||||
" alphas = np.zeros((N, M)) # alpha值\n",
|
||||
" T = M # 有几个时刻，有几个观测序列，就有几个时刻\n",
|
||||
" for t in range(T): # 遍历每一时刻，算出alpha值\n",
|
||||
" indexOfO = V.index(O[t]) # 找出序列对应的索引\n",
|
||||
" for i in range(N):\n",
|
||||
" if t == 0: # 计算初值\n",
|
||||
" alphas[i][t] = PI[t][i] * B[i][indexOfO] # P176（10.15）\n",
|
||||
" print('alpha1(%d)=p%db%db(o1)=%f' % (i, i, i, alphas[i][t]))\n",
|
||||
" else:\n",
|
||||
" alphas[i][t] = np.dot([alpha[t - 1] for alpha in alphas], [a[i] for a in A]) * B[i][\n",
|
||||
" indexOfO] # 对应P176（10.16）\n",
|
||||
" print('alpha%d(%d)=[sigma alpha%d(i)ai%d]b%d(o%d)=%f' % (t, i, t - 1, i, i, t, alphas[i][t]))\n",
|
||||
" # print(alphas)\n",
|
||||
" P = np.sum([alpha[M - 1] for alpha in alphas]) # P176(10.17)\n",
|
||||
" # alpha11 = pi[0][0] * B[0][0] #代表a1(1)\n",
|
||||
" # alpha12 = pi[0][1] * B[1][0] #代表a1(2)\n",
|
||||
" # alpha13 = pi[0][2] * B[2][0] #代表a1(3)\n",
|
||||
"\n",
|
||||
" def backward(self, Q, V, A, B, O, PI): # 后向算法\n",
|
||||
" N = len(Q) # 状态序列的大小\n",
|
||||
" M = len(O) # 观测序列的大小\n",
|
||||
" betas = np.ones((N, M)) # beta\n",
|
||||
" for i in range(N):\n",
|
||||
" print('beta%d(%d)=1' % (M, i))\n",
|
||||
" for t in range(M - 2, -1, -1):\n",
|
||||
" indexOfO = V.index(O[t + 1]) # 找出序列对应的索引\n",
|
||||
" for i in range(N):\n",
|
||||
" betas[i][t] = np.dot(np.multiply(A[i], [b[indexOfO] for b in B]), [beta[t + 1] for beta in betas])\n",
|
||||
" realT = t + 1\n",
|
||||
" realI = i + 1\n",
|
||||
" print('beta%d(%d)=[sigma a%djbj(o%d)]beta%d(j)=(' % (realT, realI, realI, realT + 1, realT + 1),\n",
|
||||
" end='')\n",
|
||||
" for j in range(N):\n",
|
||||
" print(\"%.2f*%.2f*%.2f+\" % (A[i][j], B[j][indexOfO], betas[j][t + 1]), end='')\n",
|
||||
" print(\"0)=%.3f\" % betas[i][t])\n",
|
||||
" # print(betas)\n",
|
||||
" indexOfO = V.index(O[0])\n",
|
||||
" P = np.dot(np.multiply(PI, [b[indexOfO] for b in B]), [beta[0] for beta in betas])\n",
|
||||
" print(\"P(O|lambda)=\", end=\"\")\n",
|
||||
" for i in range(N):\n",
|
||||
" print(\"%.1f*%.1f*%.5f+\" % (PI[0][i], B[i][indexOfO], betas[i][0]), end=\"\")\n",
|
||||
" print(\"0=%f\" % P)\n",
|
||||
"\n",
|
||||
" def viterbi(self, Q, V, A, B, O, PI):\n",
|
||||
" N = len(Q) # 状态序列的大小\n",
|
||||
" M = len(O) # 观测序列的大小\n",
|
||||
" deltas = np.zeros((N, M))\n",
|
||||
" psis = np.zeros((N, M))\n",
|
||||
" I = np.zeros((1, M))\n",
|
||||
" for t in range(M):\n",
|
||||
" realT = t+1\n",
|
||||
" indexOfO = V.index(O[t]) # 找出序列对应的索引\n",
|
||||
" for i in range(N):\n",
|
||||
" realI = i+1\n",
|
||||
" if t == 0:\n",
|
||||
" deltas[i][t] = PI[0][i] * B[i][indexOfO]\n",
|
||||
" psis[i][t] = 0\n",
|
||||
" print('delta1(%d)=pi%d * b%d(o1)=%.2f * %.2f=%.2f'%(realI, realI, realI, PI[0][i], B[i][indexOfO], deltas[i][t]))\n",
|
||||
" print('psis1(%d)=0' % (realI))\n",
|
||||
" else:\n",
|
||||
" deltas[i][t] = np.max(np.multiply([delta[t-1] for delta in deltas], [a[i] for a in A])) * B[i][indexOfO]\n",
|
||||
" print('delta%d(%d)=max[delta%d(j)aj%d]b%d(o%d)=%.2f*%.2f=%.5f'%(realT, realI, realT-1, realI, realI, realT, np.max(np.multiply([delta[t-1] for delta in deltas], [a[i] for a in A])), B[i][indexOfO], deltas[i][t]))\n",
|
||||
" psis[i][t] = np.argmax(np.multiply([delta[t-1] for delta in deltas], [a[i] for a in A]))\n",
|
||||
" print('psis%d(%d)=argmax[delta%d(j)aj%d]=%d' % (realT, realI, realT-1, realI, psis[i][t]))\n",
|
||||
" print(deltas)\n",
|
||||
" print(psis)\n",
|
||||
" I[0][M-1] = np.argmax([delta[M-1] for delta in deltas])\n",
|
||||
" print('i%d=argmax[deltaT(i)]=%d' % (M, I[0][M-1]+1))\n",
|
||||
" for t in range(M-2, -1, -1):\n",
|
||||
" I[0][t] = psis[int(I[0][t+1])][t+1]\n",
|
||||
" print('i%d=psis%d(i%d)=%d' % (t+1, t+2, t+2, I[0][t]+1))\n",
|
||||
" print(I)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 习题10.1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#习题10.1\n",
|
||||
"Q = [1, 2, 3]\n",
|
||||
"V = ['红', '白']\n",
|
||||
"A = [[0.5, 0.2, 0.3], [0.3, 0.5, 0.2], [0.2, 0.3, 0.5]]\n",
|
||||
"B = [[0.5, 0.5], [0.4, 0.6], [0.7, 0.3]]\n",
|
||||
"# O = ['红', '白', '红', '红', '白', '红', '白', '白']\n",
|
||||
"O = ['红', '白', '红', '白'] #习题10.1的例子\n",
|
||||
"PI = [[0.2, 0.4, 0.4]]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"delta1(1)=pi1 * b1(o1)=0.20 * 0.50=0.10\n",
|
||||
"psis1(1)=0\n",
|
||||
"delta1(2)=pi2 * b2(o1)=0.40 * 0.40=0.16\n",
|
||||
"psis1(2)=0\n",
|
||||
"delta1(3)=pi3 * b3(o1)=0.40 * 0.70=0.28\n",
|
||||
"psis1(3)=0\n",
|
||||
"delta2(1)=max[delta1(j)aj1]b1(o2)=0.06*0.50=0.02800\n",
|
||||
"psis2(1)=argmax[delta1(j)aj1]=2\n",
|
||||
"delta2(2)=max[delta1(j)aj2]b2(o2)=0.08*0.60=0.05040\n",
|
||||
"psis2(2)=argmax[delta1(j)aj2]=2\n",
|
||||
"delta2(3)=max[delta1(j)aj3]b3(o2)=0.14*0.30=0.04200\n",
|
||||
"psis2(3)=argmax[delta1(j)aj3]=2\n",
|
||||
"delta3(1)=max[delta2(j)aj1]b1(o3)=0.02*0.50=0.00756\n",
|
||||
"psis3(1)=argmax[delta2(j)aj1]=1\n",
|
||||
"delta3(2)=max[delta2(j)aj2]b2(o3)=0.03*0.40=0.01008\n",
|
||||
"psis3(2)=argmax[delta2(j)aj2]=1\n",
|
||||
"delta3(3)=max[delta2(j)aj3]b3(o3)=0.02*0.70=0.01470\n",
|
||||
"psis3(3)=argmax[delta2(j)aj3]=2\n",
|
||||
"delta4(1)=max[delta3(j)aj1]b1(o4)=0.00*0.50=0.00189\n",
|
||||
"psis4(1)=argmax[delta3(j)aj1]=0\n",
|
||||
"delta4(2)=max[delta3(j)aj2]b2(o4)=0.01*0.60=0.00302\n",
|
||||
"psis4(2)=argmax[delta3(j)aj2]=1\n",
|
||||
"delta4(3)=max[delta3(j)aj3]b3(o4)=0.01*0.30=0.00220\n",
|
||||
"psis4(3)=argmax[delta3(j)aj3]=2\n",
|
||||
"[[0.1 0.028 0.00756 0.00189 ]\n",
|
||||
" [0.16 0.0504 0.01008 0.003024]\n",
|
||||
" [0.28 0.042 0.0147 0.002205]]\n",
|
||||
"[[0. 2. 1. 0.]\n",
|
||||
" [0. 2. 1. 1.]\n",
|
||||
" [0. 2. 2. 2.]]\n",
|
||||
"i4=argmax[deltaT(i)]=2\n",
|
||||
"i3=psis4(i4)=2\n",
|
||||
"i2=psis3(i3)=2\n",
|
||||
"i1=psis2(i2)=3\n",
|
||||
"[[2. 1. 1. 1.]]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"HMM = HiddenMarkov()\n",
|
||||
"# HMM.forward(Q, V, A, B, O, PI)\n",
|
||||
"# HMM.backward(Q, V, A, B, O, PI)\n",
|
||||
"HMM.viterbi(Q, V, A, B, O, PI)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 习题10.2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"Q = [1, 2, 3]\n",
|
||||
"V = ['红', '白']\n",
|
||||
"A = [[0.5, 0.2, 0.3], [0.3, 0.5, 0.2], [0.2, 0.3, 0.5]]\n",
|
||||
"B = [[0.5, 0.5], [0.4, 0.6], [0.7, 0.3]]\n",
|
||||
"O = ['红', '白', '红', '红', '白', '红', '白', '白']\n",
|
||||
"PI = [[0.2, 0.3, 0.5]]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"alpha1(0)=p0b0b(o1)=0.100000\n",
|
||||
"alpha1(1)=p1b1b(o1)=0.120000\n",
|
||||
"alpha1(2)=p2b2b(o1)=0.350000\n",
|
||||
"alpha1(0)=[sigma alpha0(i)ai0]b0(o1)=0.078000\n",
|
||||
"alpha1(1)=[sigma alpha0(i)ai1]b1(o1)=0.111000\n",
|
||||
"alpha1(2)=[sigma alpha0(i)ai2]b2(o1)=0.068700\n",
|
||||
"alpha2(0)=[sigma alpha1(i)ai0]b0(o2)=0.043020\n",
|
||||
"alpha2(1)=[sigma alpha1(i)ai1]b1(o2)=0.036684\n",
|
||||
"alpha2(2)=[sigma alpha1(i)ai2]b2(o2)=0.055965\n",
|
||||
"alpha3(0)=[sigma alpha2(i)ai0]b0(o3)=0.021854\n",
|
||||
"alpha3(1)=[sigma alpha2(i)ai1]b1(o3)=0.017494\n",
|
||||
"alpha3(2)=[sigma alpha2(i)ai2]b2(o3)=0.033758\n",
|
||||
"alpha4(0)=[sigma alpha3(i)ai0]b0(o4)=0.011463\n",
|
||||
"alpha4(1)=[sigma alpha3(i)ai1]b1(o4)=0.013947\n",
|
||||
"alpha4(2)=[sigma alpha3(i)ai2]b2(o4)=0.008080\n",
|
||||
"alpha5(0)=[sigma alpha4(i)ai0]b0(o5)=0.005766\n",
|
||||
"alpha5(1)=[sigma alpha4(i)ai1]b1(o5)=0.004676\n",
|
||||
"alpha5(2)=[sigma alpha4(i)ai2]b2(o5)=0.007188\n",
|
||||
"alpha6(0)=[sigma alpha5(i)ai0]b0(o6)=0.002862\n",
|
||||
"alpha6(1)=[sigma alpha5(i)ai1]b1(o6)=0.003389\n",
|
||||
"alpha6(2)=[sigma alpha5(i)ai2]b2(o6)=0.001878\n",
|
||||
"alpha7(0)=[sigma alpha6(i)ai0]b0(o7)=0.001411\n",
|
||||
"alpha7(1)=[sigma alpha6(i)ai1]b1(o7)=0.001698\n",
|
||||
"alpha7(2)=[sigma alpha6(i)ai2]b2(o7)=0.000743\n",
|
||||
"beta8(0)=1\n",
|
||||
"beta8(1)=1\n",
|
||||
"beta8(2)=1\n",
|
||||
"beta7(1)=[sigma a1jbj(o8)]beta8(j)=(0.50*0.50*1.00+0.20*0.60*1.00+0.30*0.30*1.00+0)=0.460\n",
|
||||
"beta7(2)=[sigma a2jbj(o8)]beta8(j)=(0.30*0.50*1.00+0.50*0.60*1.00+0.20*0.30*1.00+0)=0.510\n",
|
||||
"beta7(3)=[sigma a3jbj(o8)]beta8(j)=(0.20*0.50*1.00+0.30*0.60*1.00+0.50*0.30*1.00+0)=0.430\n",
|
||||
"beta6(1)=[sigma a1jbj(o7)]beta7(j)=(0.50*0.50*0.46+0.20*0.60*0.51+0.30*0.30*0.43+0)=0.215\n",
|
||||
"beta6(2)=[sigma a2jbj(o7)]beta7(j)=(0.30*0.50*0.46+0.50*0.60*0.51+0.20*0.30*0.43+0)=0.248\n",
|
||||
"beta6(3)=[sigma a3jbj(o7)]beta7(j)=(0.20*0.50*0.46+0.30*0.60*0.51+0.50*0.30*0.43+0)=0.202\n",
|
||||
"beta5(1)=[sigma a1jbj(o6)]beta6(j)=(0.50*0.50*0.21+0.20*0.40*0.25+0.30*0.70*0.20+0)=0.116\n",
|
||||
"beta5(2)=[sigma a2jbj(o6)]beta6(j)=(0.30*0.50*0.21+0.50*0.40*0.25+0.20*0.70*0.20+0)=0.110\n",
|
||||
"beta5(3)=[sigma a3jbj(o6)]beta6(j)=(0.20*0.50*0.21+0.30*0.40*0.25+0.50*0.70*0.20+0)=0.122\n",
|
||||
"beta4(1)=[sigma a1jbj(o5)]beta5(j)=(0.50*0.50*0.12+0.20*0.60*0.11+0.30*0.30*0.12+0)=0.053\n",
|
||||
"beta4(2)=[sigma a2jbj(o5)]beta5(j)=(0.30*0.50*0.12+0.50*0.60*0.11+0.20*0.30*0.12+0)=0.058\n",
|
||||
"beta4(3)=[sigma a3jbj(o5)]beta5(j)=(0.20*0.50*0.12+0.30*0.60*0.11+0.50*0.30*0.12+0)=0.050\n",
|
||||
"beta3(1)=[sigma a1jbj(o4)]beta4(j)=(0.50*0.50*0.05+0.20*0.40*0.06+0.30*0.70*0.05+0)=0.028\n",
|
||||
"beta3(2)=[sigma a2jbj(o4)]beta4(j)=(0.30*0.50*0.05+0.50*0.40*0.06+0.20*0.70*0.05+0)=0.026\n",
|
||||
"beta3(3)=[sigma a3jbj(o4)]beta4(j)=(0.20*0.50*0.05+0.30*0.40*0.06+0.50*0.70*0.05+0)=0.030\n",
|
||||
"beta2(1)=[sigma a1jbj(o3)]beta3(j)=(0.50*0.50*0.03+0.20*0.40*0.03+0.30*0.70*0.03+0)=0.015\n",
|
||||
"beta2(2)=[sigma a2jbj(o3)]beta3(j)=(0.30*0.50*0.03+0.50*0.40*0.03+0.20*0.70*0.03+0)=0.014\n",
|
||||
"beta2(3)=[sigma a3jbj(o3)]beta3(j)=(0.20*0.50*0.03+0.30*0.40*0.03+0.50*0.70*0.03+0)=0.016\n",
|
||||
"beta1(1)=[sigma a1jbj(o2)]beta2(j)=(0.50*0.50*0.02+0.20*0.60*0.01+0.30*0.30*0.02+0)=0.007\n",
|
||||
"beta1(2)=[sigma a2jbj(o2)]beta2(j)=(0.30*0.50*0.02+0.50*0.60*0.01+0.20*0.30*0.02+0)=0.007\n",
|
||||
"beta1(3)=[sigma a3jbj(o2)]beta2(j)=(0.20*0.50*0.02+0.30*0.60*0.01+0.50*0.30*0.02+0)=0.006\n",
|
||||
"P(O|lambda)=0.2*0.5*0.00698+0.3*0.4*0.00741+0.5*0.7*0.00647+0=0.003852\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"HMM.forward(Q, V, A, B, O, PI)\n",
|
||||
"HMM.backward(Q, V, A, B, O, PI)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"原文代码作者：https://blog.csdn.net/GrinAndBearIt/article/details/79229803\n",
|
||||
"\n",
|
||||
"中文注释制作：机器学习初学者(微信公众号：ID:ai-start-com)\n",
|
||||
"\n",
|
||||
"配置环境：python 3.6\n",
|
||||
"\n",
|
||||
"代码全部测试通过。\n",
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 第11章 条件随机场\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 例11.1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from numpy import *"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"24.532530197109345\n",
|
||||
"24.532530197109352\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"#这里定义T为转移矩阵列代表前一个y(ij)代表由状态i转到状态j的概率,Tx矩阵x对应于时间序列\n",
|
||||
"#这里将书上的转移特征转换为如下以时间轴为区别的三个多维列表，维度为输出的维度\n",
|
||||
"T1=[[0.6,1],[1,0]];T2=[[0,1],[1,0.2]]\n",
|
||||
"#将书上的状态特征同样转换成列表,第一个是为y1的未规划概率，第二个为y2的未规划概率\n",
|
||||
"S0=[1,0.5];S1=[0.8,0.5];S2=[0.8,0.5]\n",
|
||||
"Y=[1,2,2] #即书上例一需要计算的非规划条件概率的标记序列\n",
|
||||
"Y=array(Y)-1 #这里为了将数与索引相对应即从零开始\n",
|
||||
"P=exp(S0[Y[0]])\n",
|
||||
"for i in range(1,len(Y)):\n",
|
||||
" P *= exp((eval('S%d' % i)[Y[i]])+eval('T%d' % i)[Y[i-1]][Y[i]])\n",
|
||||
"print(P)\n",
|
||||
"print(exp(3.2))\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 例11.2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"非规范化概率 24.532530197109345\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"#这里根据例11.2的启发整合为一个矩阵\n",
|
||||
"F0=S0;F1=T1+array(S1*len(T1)).reshape(shape(T1));F2=T2+array(S2*len(T2)).reshape(shape(T2))\n",
|
||||
"Y=[1,2,2] #即书上例一需要计算的非规划条件概率的标记序列\n",
|
||||
"Y=array(Y)-1\n",
|
||||
"\n",
|
||||
"P=exp(F0[Y[0]])\n",
|
||||
"Sum=P\n",
|
||||
"for i in range(1,len(Y)):\n",
|
||||
" PIter=exp((eval('F%d' % i)[Y[i-1]][Y[i]]))\n",
|
||||
" P *= PIter\n",
|
||||
" Sum += PIter\n",
|
||||
"print('非规范化概率',P)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"原文代码作者：https://blog.csdn.net/GrinAndBearIt/article/details/79229803\n",
|
||||
"\n",
|
||||
"中文注释制作：机器学习初学者(微信公众号：ID:ai-start-com)\n",
|
||||
"\n",
|
||||
"配置环境：python 3.6\n",
|
||||
"\n",
|
||||
"代码全部测试通过。\n",
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 第11章 条件随机场\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 例11.1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from numpy import *"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"24.532530197109345\n",
|
||||
"24.532530197109352\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"#这里定义T为转移矩阵列代表前一个y(ij)代表由状态i转到状态j的概率,Tx矩阵x对应于时间序列\n",
|
||||
"#这里将书上的转移特征转换为如下以时间轴为区别的三个多维列表，维度为输出的维度\n",
|
||||
"T1=[[0.6,1],[1,0]];T2=[[0,1],[1,0.2]]\n",
|
||||
"#将书上的状态特征同样转换成列表,第一个是为y1的未规划概率，第二个为y2的未规划概率\n",
|
||||
"S0=[1,0.5];S1=[0.8,0.5];S2=[0.8,0.5]\n",
|
||||
"Y=[1,2,2] #即书上例一需要计算的非规划条件概率的标记序列\n",
|
||||
"Y=array(Y)-1 #这里为了将数与索引相对应即从零开始\n",
|
||||
"P=exp(S0[Y[0]])\n",
|
||||
"for i in range(1,len(Y)):\n",
|
||||
" P *= exp((eval('S%d' % i)[Y[i]])+eval('T%d' % i)[Y[i-1]][Y[i]])\n",
|
||||
"print(P)\n",
|
||||
"print(exp(3.2))\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 例11.2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"非规范化概率 24.532530197109345\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"#这里根据例11.2的启发整合为一个矩阵\n",
|
||||
"F0=S0;F1=T1+array(S1*len(T1)).reshape(shape(T1));F2=T2+array(S2*len(T2)).reshape(shape(T2))\n",
|
||||
"Y=[1,2,2] #即书上例一需要计算的非规划条件概率的标记序列\n",
|
||||
"Y=array(Y)-1\n",
|
||||
"\n",
|
||||
"P=exp(F0[Y[0]])\n",
|
||||
"Sum=P\n",
|
||||
"for i in range(1,len(Y)):\n",
|
||||
" PIter=exp((eval('F%d' % i)[Y[i-1]][Y[i]]))\n",
|
||||
" P *= PIter\n",
|
||||
" Sum += PIter\n",
|
||||
"print('非规范化概率',P)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
+366
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,123 @@
|
||||
import numpy as np
|
||||
from math import sqrt
|
||||
import pandas as pd
|
||||
from sklearn.datasets import load_iris
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
iris = load_iris()
|
||||
df = pd.DataFrame(iris.data, columns=iris.feature_names)
|
||||
df['label'] = iris.target
|
||||
df.columns = ['sepal length', 'sepal width', 'petal length', 'petal width', 'label']
|
||||
|
||||
data = np.array(df.iloc[:100, [0, 1, -1]])
|
||||
train, test = train_test_split(data, test_size=0.1)
|
||||
x0 = np.array([x0 for i, x0 in enumerate(train) if train[i][-1] == 0])
|
||||
x1 = np.array([x1 for i, x1 in enumerate(train) if train[i][-1] == 1])
|
||||
|
||||
|
||||
def show_train():
|
||||
plt.scatter(x0[:, 0], x0[:, 1], c='pink', label='[0]')
|
||||
plt.scatter(x1[:, 0], x1[:, 1], c='orange', label='[1]')
|
||||
plt.xlabel('sepal length')
|
||||
plt.ylabel('sepal width')
|
||||
|
||||
|
||||
class Node:
|
||||
def __init__(self, data, depth=0, lchild=None, rchild=None):
|
||||
self.data = data
|
||||
self.depth = depth
|
||||
self.lchild = lchild
|
||||
self.rchild = rchild
|
||||
|
||||
|
||||
class KdTree:
|
||||
def __init__(self):
|
||||
self.KdTree = None
|
||||
self.n = 0
|
||||
self.nearest = None
|
||||
|
||||
def create(self, dataSet, depth=0):
|
||||
if len(dataSet) > 0:
|
||||
m, n = np.shape(dataSet)
|
||||
self.n = n - 1
|
||||
axis = depth % self.n
|
||||
mid = int(m / 2)
|
||||
dataSetcopy = sorted(dataSet, key=lambda x: x[axis])
|
||||
node = Node(dataSetcopy[mid], depth)
|
||||
if depth == 0:
|
||||
self.KdTree = node
|
||||
node.lchild = self.create(dataSetcopy[:mid], depth+1)
|
||||
node.rchild = self.create(dataSetcopy[mid+1:], depth+1)
|
||||
return node
|
||||
return None
|
||||
|
||||
def preOrder(self, node):
|
||||
if node is not None:
|
||||
print(node.depth, node.data)
|
||||
self.preOrder(node.lchild)
|
||||
self.preOrder(node.rchild)
|
||||
|
||||
def search(self, x, count=1):
|
||||
nearest = []
|
||||
for i in range(count):
|
||||
nearest.append([-1, None])
|
||||
self.nearest = np.array(nearest)
|
||||
|
||||
def recurve(node):
|
||||
if node is not None:
|
||||
axis = node.depth % self.n
|
||||
daxis = x[axis] - node.data[axis]
|
||||
if daxis < 0:
|
||||
recurve(node.lchild)
|
||||
else:
|
||||
recurve(node.rchild)
|
||||
|
||||
dist = sqrt(sum((p1 - p2) ** 2 for p1, p2 in zip(x, node.data)))
|
||||
for i, d in enumerate(self.nearest):
|
||||
if d[0] < 0 or dist < d[0]:
|
||||
self.nearest = np.insert(self.nearest, i, [dist, node], axis=0)
|
||||
self.nearest = self.nearest[:-1]
|
||||
break
|
||||
|
||||
n = list(self.nearest[:, 0]).count(-1)
|
||||
if self.nearest[-n-1, 0] > abs(daxis):
|
||||
if daxis < 0:
|
||||
recurve(node.rchild)
|
||||
else:
|
||||
recurve(node.lchild)
|
||||
|
||||
recurve(self.KdTree)
|
||||
|
||||
knn = self.nearest[:, 1]
|
||||
belong = []
|
||||
for i in knn:
|
||||
belong.append(i.data[-1])
|
||||
b = max(set(belong), key=belong.count)
|
||||
|
||||
return self.nearest, b
|
||||
|
||||
|
||||
kdt = KdTree()
|
||||
kdt.create(train)
|
||||
kdt.preOrder(kdt.KdTree)
|
||||
|
||||
score = 0
|
||||
for x in test:
|
||||
input('press Enter to show next:')
|
||||
show_train()
|
||||
plt.scatter(x[0], x[1], c='red', marker='x') # 测试点
|
||||
near, belong = kdt.search(x[:-1], 5) # 设置临近点的个数
|
||||
if belong == x[-1]:
|
||||
score += 1
|
||||
print("test:")
|
||||
print(x, "predict:", belong)
|
||||
print("nearest:")
|
||||
for n in near:
|
||||
print(n[1].data, "dist:", n[0])
|
||||
plt.scatter(n[1].data[0], n[1].data[1], c='green', marker='+') # k个最近邻点
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
score /= len(test)
|
||||
print("score:", score)
|
||||
+1230
File diff suppressed because one or more lines are too long
@@ -0,0 +1,372 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"原文代码作者：https://github.com/wzyonggege/statistical-learning-method\n",
|
||||
"\n",
|
||||
"中文注释制作：机器学习初学者(微信公众号：ID:ai-start-com)\n",
|
||||
"\n",
|
||||
"配置环境：python 3.6\n",
|
||||
"\n",
|
||||
"代码全部测试通过。\n",
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 第4章 朴素贝叶斯"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"基于贝叶斯定理与特征条件独立假设的分类方法。\n",
|
||||
"\n",
|
||||
"模型：\n",
|
||||
"\n",
|
||||
"- 高斯模型\n",
|
||||
"- 多项式模型\n",
|
||||
"- 伯努利模型"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"from sklearn.datasets import load_iris\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"\n",
|
||||
"from collections import Counter\n",
|
||||
"import math"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# data\n",
|
||||
"def create_data():\n",
|
||||
" iris = load_iris()\n",
|
||||
" df = pd.DataFrame(iris.data, columns=iris.feature_names)\n",
|
||||
" df['label'] = iris.target\n",
|
||||
" df.columns = ['sepal length', 'sepal width', 'petal length', 'petal width', 'label']\n",
|
||||
" data = np.array(df.iloc[:100, :])\n",
|
||||
" # print(data)\n",
|
||||
" return data[:,:-1], data[:,-1]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"X, y = create_data()\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"(array([4.6, 3.4, 1.4, 0.3]), 0.0)"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"X_test[0], y_test[0]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"参考：https://machinelearningmastery.com/naive-bayes-classifier-scratch-python/\n",
|
||||
"\n",
|
||||
"## GaussianNB 高斯朴素贝叶斯\n",
|
||||
"\n",
|
||||
"特征的可能性被假设为高斯\n",
|
||||
"\n",
|
||||
"概率密度函数：\n",
|
||||
"$$P(x_i | y_k)=\\frac{1}{\\sqrt{2\\pi\\sigma^2_{yk}}}exp(-\\frac{(x_i-\\mu_{yk})^2}{2\\sigma^2_{yk}})$$\n",
|
||||
"\n",
|
||||
"数学期望(mean)：$\\mu$，方差：$\\sigma^2=\\frac{\\sum(X-\\mu)^2}{N}$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class NaiveBayes:\n",
|
||||
" def __init__(self):\n",
|
||||
" self.model = None\n",
|
||||
"\n",
|
||||
" # 数学期望\n",
|
||||
" @staticmethod\n",
|
||||
" def mean(X):\n",
|
||||
" return sum(X) / float(len(X))\n",
|
||||
"\n",
|
||||
" # 标准差（方差）\n",
|
||||
" def stdev(self, X):\n",
|
||||
" avg = self.mean(X)\n",
|
||||
" return math.sqrt(sum([pow(x-avg, 2) for x in X]) / float(len(X)))\n",
|
||||
"\n",
|
||||
" # 概率密度函数\n",
|
||||
" def gaussian_probability(self, x, mean, stdev):\n",
|
||||
" exponent = math.exp(-(math.pow(x-mean,2)/(2*math.pow(stdev,2))))\n",
|
||||
" return (1 / (math.sqrt(2*math.pi) * stdev)) * exponent\n",
|
||||
"\n",
|
||||
" # 处理X_train\n",
|
||||
" def summarize(self, train_data):\n",
|
||||
" summaries = [(self.mean(i), self.stdev(i)) for i in zip(*train_data)]\n",
|
||||
" return summaries\n",
|
||||
"\n",
|
||||
" # 分类别求出数学期望和标准差\n",
|
||||
" def fit(self, X, y):\n",
|
||||
" labels = list(set(y))\n",
|
||||
" data = {label:[] for label in labels}\n",
|
||||
" for f, label in zip(X, y):\n",
|
||||
" data[label].append(f)\n",
|
||||
" self.model = {label: self.summarize(value) for label, value in data.items()}\n",
|
||||
" return 'gaussianNB train done!'\n",
|
||||
"\n",
|
||||
" # 计算概率\n",
|
||||
" def calculate_probabilities(self, input_data):\n",
|
||||
" # summaries:{0.0: [(5.0, 0.37),(3.42, 0.40)], 1.0: [(5.8, 0.449),(2.7, 0.27)]}\n",
|
||||
" # input_data:[1.1, 2.2]\n",
|
||||
" probabilities = {}\n",
|
||||
" for label, value in self.model.items():\n",
|
||||
" probabilities[label] = 1\n",
|
||||
" for i in range(len(value)):\n",
|
||||
" mean, stdev = value[i]\n",
|
||||
" probabilities[label] *= self.gaussian_probability(input_data[i], mean, stdev)\n",
|
||||
" return probabilities\n",
|
||||
"\n",
|
||||
" # 类别\n",
|
||||
" def predict(self, X_test):\n",
|
||||
" # {0.0: 2.9680340789325763e-27, 1.0: 3.5749783019849535e-26}\n",
|
||||
" label = sorted(self.calculate_probabilities(X_test).items(), key=lambda x: x[-1])[-1][0]\n",
|
||||
" return label\n",
|
||||
"\n",
|
||||
" def score(self, X_test, y_test):\n",
|
||||
" right = 0\n",
|
||||
" for X, y in zip(X_test, y_test):\n",
|
||||
" label = self.predict(X)\n",
|
||||
" if label == y:\n",
|
||||
" right += 1\n",
|
||||
"\n",
|
||||
" return right / float(len(X_test))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = NaiveBayes()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'gaussianNB train done!'"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.fit(X_train, y_train)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"0.0\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(model.predict([4.4, 3.2, 1.3, 0.2]))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"1.0"
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model.score(X_test, y_test)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"source": [
|
||||
"scikit-learn实例\n",
|
||||
"\n",
|
||||
"# sklearn.naive_bayes"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.naive_bayes import GaussianNB"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"GaussianNB(priors=None)"
|
||||
]
|
||||
},
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"clf = GaussianNB()\n",
|
||||
"clf.fit(X_train, y_train)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"1.0"
|
||||
]
|
||||
},
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"clf.score(X_test, y_test)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"array([0.])"
|
||||
]
|
||||
},
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"clf.predict([[4.4, 3.2, 1.3, 0.2]])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.naive_bayes import BernoulliNB, MultinomialNB # 伯努利模型和多项式模型"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
+886
@@ -0,0 +1,886 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"原文代码作者：https://github.com/wzyonggege/statistical-learning-method\n",
|
||||
"\n",
|
||||
"中文注释制作：机器学习初学者(微信公众号：ID:ai-start-com)\n",
|
||||
"\n",
|
||||
"配置环境：python 3.6\n",
|
||||
"\n",
|
||||
"代码全部测试通过。\n",
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 第5章 决策树"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"- ID3（基于信息增益）\n",
|
||||
"- C4.5（基于信息增益比）\n",
|
||||
"- CART（gini指数）"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### entropy：$H(x) = -\\sum_{i=1}^{n}p_i\\log{p_i}$\n",
|
||||
"\n",
|
||||
"#### conditional entropy: $H(X|Y)=\\sum{P(X|Y)}\\log{P(X|Y)}$\n",
|
||||
"\n",
|
||||
"#### information gain : $g(D, A)=H(D)-H(D|A)$\n",
|
||||
"\n",
|
||||
"#### information gain ratio: $g_R(D, A) = \\frac{g(D,A)}{H(A)}$\n",
|
||||
"\n",
|
||||
"#### gini index:$Gini(D)=\\sum_{k=1}^{K}p_k\\log{p_k}=1-\\sum_{k=1}^{K}p_k^2$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"from sklearn.datasets import load_iris\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"\n",
|
||||
"from collections import Counter\n",
|
||||
"import math\n",
|
||||
"from math import log\n",
|
||||
"\n",
|
||||
"import pprint"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 书上题目5.1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 书上题目5.1\n",
|
||||
"def create_data():\n",
|
||||
" datasets = [['青年', '否', '否', '一般', '否'],\n",
|
||||
" ['青年', '否', '否', '好', '否'],\n",
|
||||
" ['青年', '是', '否', '好', '是'],\n",
|
||||
" ['青年', '是', '是', '一般', '是'],\n",
|
||||
" ['青年', '否', '否', '一般', '否'],\n",
|
||||
" ['中年', '否', '否', '一般', '否'],\n",
|
||||
" ['中年', '否', '否', '好', '否'],\n",
|
||||
" ['中年', '是', '是', '好', '是'],\n",
|
||||
" ['中年', '否', '是', '非常好', '是'],\n",
|
||||
" ['中年', '否', '是', '非常好', '是'],\n",
|
||||
" ['老年', '否', '是', '非常好', '是'],\n",
|
||||
" ['老年', '否', '是', '好', '是'],\n",
|
||||
" ['老年', '是', '否', '好', '是'],\n",
|
||||
" ['老年', '是', '否', '非常好', '是'],\n",
|
||||
" ['老年', '否', '否', '一般', '否'],\n",
|
||||
" ]\n",
|
||||
" labels = [u'年龄', u'有工作', u'有自己的房子', u'信贷情况', u'类别']\n",
|
||||
" # 返回数据集和每个维度的名称\n",
|
||||
" return datasets, labels"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"datasets, labels = create_data()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"train_data = pd.DataFrame(datasets, columns=labels)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>年龄</th>\n",
|
||||
" <th>有工作</th>\n",
|
||||
" <th>有自己的房子</th>\n",
|
||||
" <th>信贷情况</th>\n",
|
||||
" <th>类别</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>青年</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>一般</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>青年</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>好</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>青年</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>好</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>青年</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" <td>一般</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>青年</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>一般</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>中年</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>一般</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>中年</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>好</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>中年</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" <td>好</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>中年</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" <td>非常好</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>9</th>\n",
|
||||
" <td>中年</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" <td>非常好</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>10</th>\n",
|
||||
" <td>老年</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" <td>非常好</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>11</th>\n",
|
||||
" <td>老年</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" <td>好</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>12</th>\n",
|
||||
" <td>老年</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>好</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>13</th>\n",
|
||||
" <td>老年</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>非常好</td>\n",
|
||||
" <td>是</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>14</th>\n",
|
||||
" <td>老年</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" <td>一般</td>\n",
|
||||
" <td>否</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" 年龄 有工作 有自己的房子 信贷情况 类别\n",
|
||||
"0 青年 否 否 一般 否\n",
|
||||
"1 青年 否 否 好 否\n",
|
||||
"2 青年 是 否 好 是\n",
|
||||
"3 青年 是 是 一般 是\n",
|
||||
"4 青年 否 否 一般 否\n",
|
||||
"5 中年 否 否 一般 否\n",
|
||||
"6 中年 否 否 好 否\n",
|
||||
"7 中年 是 是 好 是\n",
|
||||
"8 中年 否 是 非常好 是\n",
|
||||
"9 中年 否 是 非常好 是\n",
|
||||
"10 老年 否 是 非常好 是\n",
|
||||
"11 老年 否 是 好 是\n",
|
||||
"12 老年 是 否 好 是\n",
|
||||
"13 老年 是 否 非常好 是\n",
|
||||
"14 老年 否 否 一般 否"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"train_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 熵\n",
|
||||
"def calc_ent(datasets):\n",
|
||||
" data_length = len(datasets)\n",
|
||||
" label_count = {}\n",
|
||||
" for i in range(data_length):\n",
|
||||
" label = datasets[i][-1]\n",
|
||||
" if label not in label_count:\n",
|
||||
" label_count[label] = 0\n",
|
||||
" label_count[label] += 1\n",
|
||||
" ent = -sum([(p/data_length)*log(p/data_length, 2) for p in label_count.values()])\n",
|
||||
" return ent\n",
|
||||
"\n",
|
||||
"# 经验条件熵\n",
|
||||
"def cond_ent(datasets, axis=0):\n",
|
||||
" data_length = len(datasets)\n",
|
||||
" feature_sets = {}\n",
|
||||
" for i in range(data_length):\n",
|
||||
" feature = datasets[i][axis]\n",
|
||||
" if feature not in feature_sets:\n",
|
||||
" feature_sets[feature] = []\n",
|
||||
" feature_sets[feature].append(datasets[i])\n",
|
||||
" cond_ent = sum([(len(p)/data_length)*calc_ent(p) for p in feature_sets.values()])\n",
|
||||
" return cond_ent\n",
|
||||
"\n",
|
||||
"# 信息增益\n",
|
||||
"def info_gain(ent, cond_ent):\n",
|
||||
" return ent - cond_ent\n",
|
||||
"\n",
|
||||
"def info_gain_train(datasets):\n",
|
||||
" count = len(datasets[0]) - 1\n",
|
||||
" ent = calc_ent(datasets)\n",
|
||||
" best_feature = []\n",
|
||||
" for c in range(count):\n",
|
||||
" c_info_gain = info_gain(ent, cond_ent(datasets, axis=c))\n",
|
||||
" best_feature.append((c, c_info_gain))\n",
|
||||
" print('特征({}) - info_gain - {:.3f}'.format(labels[c], c_info_gain))\n",
|
||||
" # 比较大小\n",
|
||||
" best_ = max(best_feature, key=lambda x: x[-1])\n",
|
||||
" return '特征({})的信息增益最大，选择为根节点特征'.format(labels[best_[0]])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"特征(年龄) - info_gain - 0.083\n",
|
||||
"特征(有工作) - info_gain - 0.324\n",
|
||||
"特征(有自己的房子) - info_gain - 0.420\n",
|
||||
"特征(信贷情况) - info_gain - 0.363\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'特征(有自己的房子)的信息增益最大，选择为根节点特征'"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"info_gain_train(np.array(datasets))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"source": [
|
||||
"---\n",
|
||||
"\n",
|
||||
"利用ID3算法生成决策树，例5.3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 定义节点类 二叉树\n",
|
||||
"class Node:\n",
|
||||
" def __init__(self, root=True, label=None, feature_name=None, feature=None):\n",
|
||||
" self.root = root\n",
|
||||
" self.label = label\n",
|
||||
" self.feature_name = feature_name\n",
|
||||
" self.feature = feature\n",
|
||||
" self.tree = {}\n",
|
||||
" self.result = {'label:': self.label, 'feature': self.feature, 'tree': self.tree}\n",
|
||||
"\n",
|
||||
" def __repr__(self):\n",
|
||||
" return '{}'.format(self.result)\n",
|
||||
"\n",
|
||||
" def add_node(self, val, node):\n",
|
||||
" self.tree[val] = node\n",
|
||||
"\n",
|
||||
" def predict(self, features):\n",
|
||||
" if self.root is True:\n",
|
||||
" return self.label\n",
|
||||
" return self.tree[features[self.feature]].predict(features)\n",
|
||||
" \n",
|
||||
"class DTree:\n",
|
||||
" def __init__(self, epsilon=0.1):\n",
|
||||
" self.epsilon = epsilon\n",
|
||||
" self._tree = {}\n",
|
||||
"\n",
|
||||
" # 熵\n",
|
||||
" @staticmethod\n",
|
||||
" def calc_ent(datasets):\n",
|
||||
" data_length = len(datasets)\n",
|
||||
" label_count = {}\n",
|
||||
" for i in range(data_length):\n",
|
||||
" label = datasets[i][-1]\n",
|
||||
" if label not in label_count:\n",
|
||||
" label_count[label] = 0\n",
|
||||
" label_count[label] += 1\n",
|
||||
" ent = -sum([(p/data_length)*log(p/data_length, 2) for p in label_count.values()])\n",
|
||||
" return ent\n",
|
||||
"\n",
|
||||
" # 经验条件熵\n",
|
||||
" def cond_ent(self, datasets, axis=0):\n",
|
||||
" data_length = len(datasets)\n",
|
||||
" feature_sets = {}\n",
|
||||
" for i in range(data_length):\n",
|
||||
" feature = datasets[i][axis]\n",
|
||||
" if feature not in feature_sets:\n",
|
||||
" feature_sets[feature] = []\n",
|
||||
" feature_sets[feature].append(datasets[i])\n",
|
||||
" cond_ent = sum([(len(p)/data_length)*self.calc_ent(p) for p in feature_sets.values()])\n",
|
||||
" return cond_ent\n",
|
||||
"\n",
|
||||
" # 信息增益\n",
|
||||
" @staticmethod\n",
|
||||
" def info_gain(ent, cond_ent):\n",
|
||||
" return ent - cond_ent\n",
|
||||
"\n",
|
||||
" def info_gain_train(self, datasets):\n",
|
||||
" count = len(datasets[0]) - 1\n",
|
||||
" ent = self.calc_ent(datasets)\n",
|
||||
" best_feature = []\n",
|
||||
" for c in range(count):\n",
|
||||
" c_info_gain = self.info_gain(ent, self.cond_ent(datasets, axis=c))\n",
|
||||
" best_feature.append((c, c_info_gain))\n",
|
||||
" # 比较大小\n",
|
||||
" best_ = max(best_feature, key=lambda x: x[-1])\n",
|
||||
" return best_\n",
|
||||
"\n",
|
||||
" def train(self, train_data):\n",
|
||||
" \"\"\"\n",
|
||||
" input:数据集D(DataFrame格式)，特征集A，阈值eta\n",
|
||||
" output:决策树T\n",
|
||||
" \"\"\"\n",
|
||||
" _, y_train, features = train_data.iloc[:, :-1], train_data.iloc[:, -1], train_data.columns[:-1]\n",
|
||||
" # 1,若D中实例属于同一类Ck，则T为单节点树，并将类Ck作为结点的类标记，返回T\n",
|
||||
" if len(y_train.value_counts()) == 1:\n",
|
||||
" return Node(root=True,\n",
|
||||
" label=y_train.iloc[0])\n",
|
||||
"\n",
|
||||
" # 2, 若A为空，则T为单节点树，将D中实例树最大的类Ck作为该节点的类标记，返回T\n",
|
||||
" if len(features) == 0:\n",
|
||||
" return Node(root=True, label=y_train.value_counts().sort_values(ascending=False).index[0])\n",
|
||||
"\n",
|
||||
" # 3,计算最大信息增益 同5.1,Ag为信息增益最大的特征\n",
|
||||
" max_feature, max_info_gain = self.info_gain_train(np.array(train_data))\n",
|
||||
" max_feature_name = features[max_feature]\n",
|
||||
"\n",
|
||||
" # 4,Ag的信息增益小于阈值eta,则置T为单节点树，并将D中是实例数最大的类Ck作为该节点的类标记，返回T\n",
|
||||
" if max_info_gain < self.epsilon:\n",
|
||||
" return Node(root=True, label=y_train.value_counts().sort_values(ascending=False).index[0])\n",
|
||||
"\n",
|
||||
" # 5,构建Ag子集\n",
|
||||
" node_tree = Node(root=False, feature_name=max_feature_name, feature=max_feature)\n",
|
||||
"\n",
|
||||
" feature_list = train_data[max_feature_name].value_counts().index\n",
|
||||
" for f in feature_list:\n",
|
||||
" sub_train_df = train_data.loc[train_data[max_feature_name] == f].drop([max_feature_name], axis=1)\n",
|
||||
"\n",
|
||||
" # 6, 递归生成树\n",
|
||||
" sub_tree = self.train(sub_train_df)\n",
|
||||
" node_tree.add_node(f, sub_tree)\n",
|
||||
"\n",
|
||||
" # pprint.pprint(node_tree.tree)\n",
|
||||
" return node_tree\n",
|
||||
"\n",
|
||||
" def fit(self, train_data):\n",
|
||||
" self._tree = self.train(train_data)\n",
|
||||
" return self._tree\n",
|
||||
"\n",
|
||||
" def predict(self, X_test):\n",
|
||||
" return self._tree.predict(X_test)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"datasets, labels = create_data()\n",
|
||||
"data_df = pd.DataFrame(datasets, columns=labels)\n",
|
||||
"dt = DTree()\n",
|
||||
"tree = dt.fit(data_df)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'label:': None, 'feature': 2, 'tree': {'否': {'label:': None, 'feature': 1, 'tree': {'否': {'label:': '否', 'feature': None, 'tree': {}}, '是': {'label:': '是', 'feature': None, 'tree': {}}}}, '是': {'label:': '是', 'feature': None, 'tree': {}}}}"
|
||||
]
|
||||
},
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tree"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'否'"
|
||||
]
|
||||
},
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"dt.predict(['老年', '否', '否', '一般'])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"---\n",
|
||||
"\n",
|
||||
"## sklearn.tree.DecisionTreeClassifier\n",
|
||||
"\n",
|
||||
"### criterion : string, optional (default=”gini”)\n",
|
||||
"The function to measure the quality of a split. Supported criteria are “gini” for the Gini impurity and “entropy” for the information gain."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# data\n",
|
||||
"def create_data():\n",
|
||||
" iris = load_iris()\n",
|
||||
" df = pd.DataFrame(iris.data, columns=iris.feature_names)\n",
|
||||
" df['label'] = iris.target\n",
|
||||
" df.columns = ['sepal length', 'sepal width', 'petal length', 'petal width', 'label']\n",
|
||||
" data = np.array(df.iloc[:100, [0, 1, -1]])\n",
|
||||
" # print(data)\n",
|
||||
" return data[:,:2], data[:,-1]\n",
|
||||
"\n",
|
||||
"X, y = create_data()\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.tree import DecisionTreeClassifier\n",
|
||||
"\n",
|
||||
"from sklearn.tree import export_graphviz\n",
|
||||
"import graphviz"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
|
||||
" max_features=None, max_leaf_nodes=None,\n",
|
||||
" min_impurity_decrease=0.0, min_impurity_split=None,\n",
|
||||
" min_samples_leaf=1, min_samples_split=2,\n",
|
||||
" min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
|
||||
" splitter='best')"
|
||||
]
|
||||
},
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"clf = DecisionTreeClassifier()\n",
|
||||
"clf.fit(X_train, y_train,)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"1.0"
|
||||
]
|
||||
},
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"clf.score(X_test, y_test)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tree_pic = export_graphviz(clf, out_file=\"mytree.pdf\")\n",
|
||||
"with open('mytree.pdf') as f:\n",
|
||||
" dot_graph = f.read()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"image/svg+xml": [
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\r\n",
|
||||
"<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
|
||||
" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
|
||||
"<!-- Generated by graphviz version 2.38.0 (20140413.2041)\r\n",
|
||||
" -->\r\n",
|
||||
"<!-- Title: Tree Pages: 1 -->\r\n",
|
||||
"<svg width=\"550pt\" height=\"477pt\"\r\n",
|
||||
" viewBox=\"0.00 0.00 550.00 477.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
|
||||
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 473)\">\r\n",
|
||||
"<title>Tree</title>\r\n",
|
||||
"<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-473 546,-473 546,4 -4,4\"/>\r\n",
|
||||
"<!-- 0 -->\r\n",
|
||||
"<g id=\"node1\" class=\"node\"><title>0</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"377.5,-469 273.5,-469 273.5,-401 377.5,-401 377.5,-469\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"325.5\" y=\"-453.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">X[0] <= 5.45</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"325.5\" y=\"-438.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.5</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"325.5\" y=\"-423.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 70</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"325.5\" y=\"-408.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [35, 35]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 1 -->\r\n",
|
||||
"<g id=\"node2\" class=\"node\"><title>1</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"316.5,-365 218.5,-365 218.5,-297 316.5,-297 316.5,-365\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"267.5\" y=\"-349.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">X[1] <= 2.85</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"267.5\" y=\"-334.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.234</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"267.5\" y=\"-319.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 37</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"267.5\" y=\"-304.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [32, 5]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 0->1 -->\r\n",
|
||||
"<g id=\"edge1\" class=\"edge\"><title>0->1</title>\r\n",
|
||||
"<path fill=\"none\" stroke=\"black\" d=\"M306.669,-400.884C301.807,-392.332 296.508,-383.013 291.423,-374.072\"/>\r\n",
|
||||
"<polygon fill=\"black\" stroke=\"black\" points=\"294.421,-372.262 286.435,-365.299 288.335,-375.722 294.421,-372.262\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"279.806\" y=\"-385.704\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">True</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 10 -->\r\n",
|
||||
"<g id=\"node11\" class=\"node\"><title>10</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"432.5,-365 334.5,-365 334.5,-297 432.5,-297 432.5,-365\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"383.5\" y=\"-349.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">X[1] <= 3.45</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"383.5\" y=\"-334.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.165</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"383.5\" y=\"-319.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 33</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"383.5\" y=\"-304.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [3, 30]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 0->10 -->\r\n",
|
||||
"<g id=\"edge10\" class=\"edge\"><title>0->10</title>\r\n",
|
||||
"<path fill=\"none\" stroke=\"black\" d=\"M344.331,-400.884C349.193,-392.332 354.492,-383.013 359.577,-374.072\"/>\r\n",
|
||||
"<polygon fill=\"black\" stroke=\"black\" points=\"362.665,-375.722 364.565,-365.299 356.579,-372.262 362.665,-375.722\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"371.194\" y=\"-385.704\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">False</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 2 -->\r\n",
|
||||
"<g id=\"node3\" class=\"node\"><title>2</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"200,-261 109,-261 109,-193 200,-193 200,-261\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"154.5\" y=\"-245.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">X[0] <= 4.7</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"154.5\" y=\"-230.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.32</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"154.5\" y=\"-215.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 5</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"154.5\" y=\"-200.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [1, 4]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 1->2 -->\r\n",
|
||||
"<g id=\"edge2\" class=\"edge\"><title>1->2</title>\r\n",
|
||||
"<path fill=\"none\" stroke=\"black\" d=\"M230.812,-296.884C220.648,-287.709 209.505,-277.65 198.95,-268.123\"/>\r\n",
|
||||
"<polygon fill=\"black\" stroke=\"black\" points=\"201.159,-265.402 191.39,-261.299 196.468,-270.598 201.159,-265.402\"/>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 5 -->\r\n",
|
||||
"<g id=\"node6\" class=\"node\"><title>5</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"316.5,-261 218.5,-261 218.5,-193 316.5,-193 316.5,-261\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"267.5\" y=\"-245.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">X[0] <= 5.35</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"267.5\" y=\"-230.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.061</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"267.5\" y=\"-215.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 32</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"267.5\" y=\"-200.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [31, 1]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 1->5 -->\r\n",
|
||||
"<g id=\"edge5\" class=\"edge\"><title>1->5</title>\r\n",
|
||||
"<path fill=\"none\" stroke=\"black\" d=\"M267.5,-296.884C267.5,-288.778 267.5,-279.982 267.5,-271.472\"/>\r\n",
|
||||
"<polygon fill=\"black\" stroke=\"black\" points=\"271,-271.299 267.5,-261.299 264,-271.299 271,-271.299\"/>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 3 -->\r\n",
|
||||
"<g id=\"node4\" class=\"node\"><title>3</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"91,-149.5 0,-149.5 0,-96.5 91,-96.5 91,-149.5\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"45.5\" y=\"-134.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.0</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"45.5\" y=\"-119.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 1</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"45.5\" y=\"-104.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [1, 0]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 2->3 -->\r\n",
|
||||
"<g id=\"edge3\" class=\"edge\"><title>2->3</title>\r\n",
|
||||
"<path fill=\"none\" stroke=\"black\" d=\"M119.111,-192.884C106.653,-181.226 92.6699,-168.141 80.2641,-156.532\"/>\r\n",
|
||||
"<polygon fill=\"black\" stroke=\"black\" points=\"82.4644,-153.797 72.7713,-149.52 77.6815,-158.908 82.4644,-153.797\"/>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 4 -->\r\n",
|
||||
"<g id=\"node5\" class=\"node\"><title>4</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"200,-149.5 109,-149.5 109,-96.5 200,-96.5 200,-149.5\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"154.5\" y=\"-134.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.0</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"154.5\" y=\"-119.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 4</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"154.5\" y=\"-104.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [0, 4]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 2->4 -->\r\n",
|
||||
"<g id=\"edge4\" class=\"edge\"><title>2->4</title>\r\n",
|
||||
"<path fill=\"none\" stroke=\"black\" d=\"M154.5,-192.884C154.5,-182.326 154.5,-170.597 154.5,-159.854\"/>\r\n",
|
||||
"<polygon fill=\"black\" stroke=\"black\" points=\"158,-159.52 154.5,-149.52 151,-159.52 158,-159.52\"/>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 6 -->\r\n",
|
||||
"<g id=\"node7\" class=\"node\"><title>6</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"316.5,-149.5 218.5,-149.5 218.5,-96.5 316.5,-96.5 316.5,-149.5\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"267.5\" y=\"-134.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.0</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"267.5\" y=\"-119.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 28</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"267.5\" y=\"-104.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [28, 0]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 5->6 -->\r\n",
|
||||
"<g id=\"edge6\" class=\"edge\"><title>5->6</title>\r\n",
|
||||
"<path fill=\"none\" stroke=\"black\" d=\"M267.5,-192.884C267.5,-182.326 267.5,-170.597 267.5,-159.854\"/>\r\n",
|
||||
"<polygon fill=\"black\" stroke=\"black\" points=\"271,-159.52 267.5,-149.52 264,-159.52 271,-159.52\"/>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 7 -->\r\n",
|
||||
"<g id=\"node8\" class=\"node\"><title>7</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"426,-157 335,-157 335,-89 426,-89 426,-157\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"380.5\" y=\"-141.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">X[1] <= 3.2</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"380.5\" y=\"-126.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.375</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"380.5\" y=\"-111.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 4</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"380.5\" y=\"-96.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [3, 1]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 5->7 -->\r\n",
|
||||
"<g id=\"edge7\" class=\"edge\"><title>5->7</title>\r\n",
|
||||
"<path fill=\"none\" stroke=\"black\" d=\"M304.188,-192.884C314.352,-183.709 325.495,-173.65 336.05,-164.123\"/>\r\n",
|
||||
"<polygon fill=\"black\" stroke=\"black\" points=\"338.532,-166.598 343.61,-157.299 333.841,-161.402 338.532,-166.598\"/>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 8 -->\r\n",
|
||||
"<g id=\"node9\" class=\"node\"><title>8</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"371,-53 280,-53 280,-0 371,-0 371,-53\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"325.5\" y=\"-37.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.0</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"325.5\" y=\"-22.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 1</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"325.5\" y=\"-7.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [0, 1]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 7->8 -->\r\n",
|
||||
"<g id=\"edge8\" class=\"edge\"><title>7->8</title>\r\n",
|
||||
"<path fill=\"none\" stroke=\"black\" d=\"M361.264,-88.9485C356.206,-80.2579 350.736,-70.8608 345.633,-62.0917\"/>\r\n",
|
||||
"<polygon fill=\"black\" stroke=\"black\" points=\"348.534,-60.1189 340.479,-53.2367 342.484,-63.6401 348.534,-60.1189\"/>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 9 -->\r\n",
|
||||
"<g id=\"node10\" class=\"node\"><title>9</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"480,-53 389,-53 389,-0 480,-0 480,-53\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"434.5\" y=\"-37.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.0</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"434.5\" y=\"-22.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 3</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"434.5\" y=\"-7.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [3, 0]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 7->9 -->\r\n",
|
||||
"<g id=\"edge9\" class=\"edge\"><title>7->9</title>\r\n",
|
||||
"<path fill=\"none\" stroke=\"black\" d=\"M399.387,-88.9485C404.353,-80.2579 409.722,-70.8608 414.733,-62.0917\"/>\r\n",
|
||||
"<polygon fill=\"black\" stroke=\"black\" points=\"417.871,-63.6557 419.793,-53.2367 411.793,-60.1826 417.871,-63.6557\"/>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 11 -->\r\n",
|
||||
"<g id=\"node12\" class=\"node\"><title>11</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"432.5,-253.5 334.5,-253.5 334.5,-200.5 432.5,-200.5 432.5,-253.5\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"383.5\" y=\"-238.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.0</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"383.5\" y=\"-223.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 30</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"383.5\" y=\"-208.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [0, 30]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 10->11 -->\r\n",
|
||||
"<g id=\"edge11\" class=\"edge\"><title>10->11</title>\r\n",
|
||||
"<path fill=\"none\" stroke=\"black\" d=\"M383.5,-296.884C383.5,-286.326 383.5,-274.597 383.5,-263.854\"/>\r\n",
|
||||
"<polygon fill=\"black\" stroke=\"black\" points=\"387,-263.52 383.5,-253.52 380,-263.52 387,-263.52\"/>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 12 -->\r\n",
|
||||
"<g id=\"node13\" class=\"node\"><title>12</title>\r\n",
|
||||
"<polygon fill=\"none\" stroke=\"black\" points=\"542,-253.5 451,-253.5 451,-200.5 542,-200.5 542,-253.5\"/>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"496.5\" y=\"-238.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">gini = 0.0</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"496.5\" y=\"-223.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 3</text>\r\n",
|
||||
"<text text-anchor=\"middle\" x=\"496.5\" y=\"-208.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [3, 0]</text>\r\n",
|
||||
"</g>\r\n",
|
||||
"<!-- 10->12 -->\r\n",
|
||||
"<g id=\"edge12\" class=\"edge\"><title>10->12</title>\r\n",
|
||||
"<path fill=\"none\" stroke=\"black\" d=\"M420.188,-296.884C433.103,-285.226 447.599,-272.141 460.46,-260.532\"/>\r\n",
|
||||
"<polygon fill=\"black\" stroke=\"black\" points=\"463.15,-262.819 468.228,-253.52 458.46,-257.622 463.15,-262.819\"/>\r\n",
|
||||
"</g>\r\n",
|
||||
"</g>\r\n",
|
||||
"</svg>\r\n"
|
||||
],
|
||||
"text/plain": [
|
||||
"<graphviz.files.Source at 0x22d67bed7b8>"
|
||||
]
|
||||
},
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"graphviz.Source(dot_graph)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from collections import Counter
|
||||
import math
|
||||
|
||||
|
||||
class Node:
|
||||
def __init__(self, x=None, label=None, y=None, data=None):
|
||||
self.label = label # label:子节点分类依据的特征
|
||||
self.x = x # x:特征
|
||||
self.child = [] # child:子节点
|
||||
self.y = y # y:类标记（叶节点才有）
|
||||
self.data = data # data:包含数据（叶节点才有）
|
||||
|
||||
def append(self, node): # 添加子节点
|
||||
self.child.append(node)
|
||||
|
||||
def predict(self, features): # 预测数据所述类
|
||||
if self.y is not None:
|
||||
return self.y
|
||||
for c in self.child:
|
||||
if c.x == features[self.label]:
|
||||
return c.predict(features)
|
||||
|
||||
|
||||
def printnode(node, depth=0): # 打印树所有节点
|
||||
if node.label is None:
|
||||
print(depth, (node.label, node.x, node.y, len(node.data)))
|
||||
else:
|
||||
print(depth, (node.label, node.x))
|
||||
for c in node.child:
|
||||
printnode(c, depth+1)
|
||||
|
||||
|
||||
class DTree:
|
||||
def __init__(self, epsilon=0, alpha=0): # 预剪枝、后剪枝参数
|
||||
self.epsilon = epsilon
|
||||
self.alpha = alpha
|
||||
self.tree = Node()
|
||||
|
||||
def prob(self, datasets): # 求概率
|
||||
datalen = len(datasets)
|
||||
labelx = set(datasets)
|
||||
p = {l: 0 for l in labelx}
|
||||
for d in datasets:
|
||||
p[d] += 1
|
||||
for i in p.items():
|
||||
p[i[0]] /= datalen
|
||||
return p
|
||||
|
||||
def calc_ent(self, datasets): # 求熵
|
||||
p = self.prob(datasets)
|
||||
ent = sum([-v * math.log(v, 2) for v in p.values()])
|
||||
return ent
|
||||
|
||||
def cond_ent(self, datasets, col): # 求条件熵
|
||||
labelx = set(datasets.iloc[col])
|
||||
p = {x: [] for x in labelx}
|
||||
for i, d in enumerate(datasets.iloc[-1]):
|
||||
p[datasets.iloc[col][i]].append(d)
|
||||
return sum([self.prob(datasets.iloc[col])[k] * self.calc_ent(p[k]) for k in p.keys()])
|
||||
|
||||
def info_gain_train(self, datasets, datalabels): # 求信息增益（互信息）
|
||||
#print('----信息增益----')
|
||||
datasets = datasets.T
|
||||
ent = self.calc_ent(datasets.iloc[-1])
|
||||
gainmax = {}
|
||||
for i in range(len(datasets) - 1):
|
||||
cond = self.cond_ent(datasets, i)
|
||||
#print(datalabels[i], ent - cond)
|
||||
gainmax[ent - cond] = i
|
||||
m = max(gainmax.keys())
|
||||
return gainmax[m], m
|
||||
|
||||
def train(self, datasets, node):
|
||||
labely = datasets.columns[-1]
|
||||
if len(datasets[labely].value_counts()) == 1:
|
||||
node.data = datasets[labely]
|
||||
node.y = datasets[labely][0]
|
||||
return
|
||||
if len(datasets.columns[:-1]) == 0:
|
||||
node.data = datasets[labely]
|
||||
node.y = datasets[labely].value_counts().index[0]
|
||||
return
|
||||
gainmaxi, gainmax = self.info_gain_train(datasets, datasets.columns)
|
||||
#print('选择特征：', gainmaxi)
|
||||
if gainmax <= self.epsilon: # 若信息增益（互信息）为0意为输入特征x完全相同而标签y相反
|
||||
node.data = datasets[labely]
|
||||
node.y = datasets[labely].value_counts().index[0]
|
||||
return
|
||||
|
||||
vc = datasets[datasets.columns[gainmaxi]].value_counts()
|
||||
for Di in vc.index:
|
||||
node.label = gainmaxi
|
||||
child = Node(Di)
|
||||
node.append(child)
|
||||
new_datasets = pd.DataFrame([list(i) for i in datasets.values if i[gainmaxi]==Di], columns=datasets.columns)
|
||||
self.train(new_datasets, child)
|
||||
|
||||
def fit(self, datasets):
|
||||
self.train(datasets, self.tree)
|
||||
|
||||
def findleaf(self, node, leaf): # 找到所有叶节点
|
||||
for t in node.child:
|
||||
if t.y is not None:
|
||||
leaf.append(t.data)
|
||||
else:
|
||||
for c in node.child:
|
||||
self.findleaf(c, leaf)
|
||||
|
||||
def findfather(self, node, errormin):
|
||||
if node.label is not None:
|
||||
cy = [c.y for c in node.child]
|
||||
if None not in cy: # 全是叶节点
|
||||
childdata = []
|
||||
for c in node.child:
|
||||
for d in list(c.data):
|
||||
childdata.append(d)
|
||||
childcounter = Counter(childdata)
|
||||
|
||||
old_child = node.child # 剪枝前先拷贝一下
|
||||
old_label = node.label
|
||||
old_y = node.y
|
||||
old_data = node.data
|
||||
|
||||
node.label = None # 剪枝
|
||||
node.y = childcounter.most_common(1)[0][0]
|
||||
node.data = childdata
|
||||
|
||||
error = self.c_error()
|
||||
if error <= errormin: # 剪枝前后损失比较
|
||||
errormin = error
|
||||
return 1
|
||||
else:
|
||||
node.child = old_child # 剪枝效果不好，则复原
|
||||
node.label = old_label
|
||||
node.y = old_y
|
||||
node.data = old_data
|
||||
else:
|
||||
re = 0
|
||||
i = 0
|
||||
while i < len(node.child):
|
||||
if_re = self.findfather(node.child[i], errormin) # 若剪过枝，则其父节点要重新检测
|
||||
if if_re == 1:
|
||||
re = 1
|
||||
elif if_re == 2:
|
||||
i -= 1
|
||||
i += 1
|
||||
if re:
|
||||
return 2
|
||||
return 0
|
||||
|
||||
def c_error(self): # 求C(T)
|
||||
leaf = []
|
||||
self.findleaf(self.tree, leaf)
|
||||
leafnum = [len(l) for l in leaf]
|
||||
ent = [self.calc_ent(l) for l in leaf]
|
||||
print("Ent:", ent)
|
||||
error = self.alpha*len(leafnum)
|
||||
for l, e in zip(leafnum, ent):
|
||||
error += l*e
|
||||
print("C(T):", error)
|
||||
return error
|
||||
|
||||
def cut(self, alpha=0): # 剪枝
|
||||
if alpha:
|
||||
self.alpha = alpha
|
||||
errormin = self.c_error()
|
||||
self.findfather(self.tree, errormin)
|
||||
|
||||
|
||||
datasets = np.array([['青年', '否', '否', '一般', '否'],
|
||||
['青年', '否', '否', '好', '否'],
|
||||
['青年', '是', '否', '好', '是'],
|
||||
['青年', '是', '是', '一般', '是'],
|
||||
['青年', '否', '否', '一般', '否'],
|
||||
['中年', '否', '否', '一般', '否'],
|
||||
['中年', '否', '否', '好', '否'],
|
||||
['中年', '是', '是', '好', '是'],
|
||||
['中年', '否', '是', '非常好', '是'],
|
||||
['中年', '否', '是', '非常好', '是'],
|
||||
['老年', '否', '是', '非常好', '是'],
|
||||
['老年', '否', '是', '好', '是'],
|
||||
['老年', '是', '否', '好', '是'],
|
||||
['老年', '是', '否', '非常好', '是'],
|
||||
['老年', '否', '否', '一般', '否'],
|
||||
['青年', '否', '否', '一般', '是']]) # 在李航原始数据上多加了最后这行数据，以便体现剪枝效果
|
||||
|
||||
datalabels = np.array(['年龄', '有工作', '有自己的房子', '信贷情况', '类别'])
|
||||
train_data = pd.DataFrame(datasets, columns=datalabels)
|
||||
test_data = ['老年', '否', '否', '一般']
|
||||
|
||||
dt = DTree(epsilon=0) # 可修改epsilon查看预剪枝效果
|
||||
dt.fit(train_data)
|
||||
|
||||
print('DTree:')
|
||||
printnode(dt.tree)
|
||||
y = dt.tree.predict(test_data)
|
||||
print('result:', y)
|
||||
|
||||
dt.cut(alpha=0.5) # 可修改正则化参数alpha查看后剪枝效果
|
||||
|
||||
print('DTree:')
|
||||
printnode(dt.tree)
|
||||
y = dt.tree.predict(test_data)
|
||||
print('result:', y)
|
||||
@@ -0,0 +1,28 @@
|
||||
digraph Tree {
|
||||
node [shape=box] ;
|
||||
0 [label="X[0] <= 5.45\ngini = 0.5\nsamples = 70\nvalue = [35, 35]"] ;
|
||||
1 [label="X[1] <= 2.85\ngini = 0.234\nsamples = 37\nvalue = [32, 5]"] ;
|
||||
0 -> 1 [labeldistance=2.5, labelangle=45, headlabel="True"] ;
|
||||
2 [label="X[0] <= 4.7\ngini = 0.32\nsamples = 5\nvalue = [1, 4]"] ;
|
||||
1 -> 2 ;
|
||||
3 [label="gini = 0.0\nsamples = 1\nvalue = [1, 0]"] ;
|
||||
2 -> 3 ;
|
||||
4 [label="gini = 0.0\nsamples = 4\nvalue = [0, 4]"] ;
|
||||
2 -> 4 ;
|
||||
5 [label="X[0] <= 5.35\ngini = 0.061\nsamples = 32\nvalue = [31, 1]"] ;
|
||||
1 -> 5 ;
|
||||
6 [label="gini = 0.0\nsamples = 28\nvalue = [28, 0]"] ;
|
||||
5 -> 6 ;
|
||||
7 [label="X[1] <= 3.2\ngini = 0.375\nsamples = 4\nvalue = [3, 1]"] ;
|
||||
5 -> 7 ;
|
||||
8 [label="gini = 0.0\nsamples = 1\nvalue = [0, 1]"] ;
|
||||
7 -> 8 ;
|
||||
9 [label="gini = 0.0\nsamples = 3\nvalue = [3, 0]"] ;
|
||||
7 -> 9 ;
|
||||
10 [label="X[1] <= 3.45\ngini = 0.165\nsamples = 33\nvalue = [3, 30]"] ;
|
||||
0 -> 10 [labeldistance=2.5, labelangle=-45, headlabel="False"] ;
|
||||
11 [label="gini = 0.0\nsamples = 30\nvalue = [0, 30]"] ;
|
||||
10 -> 11 ;
|
||||
12 [label="gini = 0.0\nsamples = 3\nvalue = [3, 0]"] ;
|
||||
10 -> 12 ;
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,122 @@
|
||||
import math
|
||||
from copy import deepcopy
|
||||
|
||||
|
||||
class MaxEntropy:
|
||||
def __init__(self, EPS=0.005):
|
||||
self._samples = []
|
||||
self._Y = set() # 标签集合，相当去去重后的y
|
||||
self._numXY = {} # key为(x,y)，value为出现次数
|
||||
self._N = 0 # 样本数
|
||||
self._Ep_ = [] # 样本分布的特征期望值
|
||||
self._xyID = {} # key记录(x,y),value记录id号
|
||||
self._n = 0 # 特征键值(x,y)的个数
|
||||
self._C = 0 # 最大特征数
|
||||
self._IDxy = {} # key为(x,y)，value为对应的id号
|
||||
self._w = []
|
||||
self._EPS = EPS # 收敛条件
|
||||
self._lastw = [] # 上一次w参数值
|
||||
|
||||
def loadData(self, dataset):
|
||||
self._samples = deepcopy(dataset)
|
||||
for items in self._samples:
|
||||
y = items[0]
|
||||
X = items[1:]
|
||||
self._Y.add(y) # 集合中y若已存在则会自动忽略
|
||||
for x in X:
|
||||
if (x, y) in self._numXY:
|
||||
self._numXY[(x, y)] += 1
|
||||
else:
|
||||
self._numXY[(x, y)] = 1
|
||||
|
||||
self._N = len(self._samples)
|
||||
self._n = len(self._numXY)
|
||||
self._C = max([len(sample)-1 for sample in self._samples])
|
||||
self._w = [0]*self._n
|
||||
self._lastw = self._w[:]
|
||||
|
||||
self._Ep_ = [0] * self._n
|
||||
for i, xy in enumerate(self._numXY): # 计算特征函数fi关于经验分布的期望
|
||||
self._Ep_[i] = self._numXY[xy]/self._N
|
||||
self._xyID[xy] = i
|
||||
self._IDxy[i] = xy
|
||||
|
||||
def _Zx(self, X): # 计算每个Z(x)值
|
||||
zx = 0
|
||||
for y in self._Y:
|
||||
ss = 0
|
||||
for x in X:
|
||||
if (x, y) in self._numXY:
|
||||
ss += self._w[self._xyID[(x, y)]]
|
||||
zx += math.exp(ss)
|
||||
return zx
|
||||
|
||||
def _model_pyx(self, y, X): # 计算每个P(y|x)
|
||||
zx = self._Zx(X)
|
||||
ss = 0
|
||||
for x in X:
|
||||
if (x, y) in self._numXY:
|
||||
ss += self._w[self._xyID[(x, y)]]
|
||||
pyx = math.exp(ss)/zx
|
||||
return pyx
|
||||
|
||||
def _model_ep(self, index): # 计算特征函数fi关于模型的期望
|
||||
x, y = self._IDxy[index]
|
||||
ep = 0
|
||||
for sample in self._samples:
|
||||
if x not in sample:
|
||||
continue
|
||||
pyx = self._model_pyx(y, sample)
|
||||
ep += pyx/self._N
|
||||
return ep
|
||||
|
||||
def _convergence(self): # 判断是否全部收敛
|
||||
for last, now in zip(self._lastw, self._w):
|
||||
if abs(last - now) >= self._EPS:
|
||||
return False
|
||||
return True
|
||||
|
||||
def predict(self, X): # 计算预测概率
|
||||
Z = self._Zx(X)
|
||||
result = {}
|
||||
for y in self._Y:
|
||||
ss = 0
|
||||
for x in X:
|
||||
if (x, y) in self._numXY:
|
||||
ss += self._w[self._xyID[(x, y)]]
|
||||
pyx = math.exp(ss)/Z
|
||||
result[y] = pyx
|
||||
return result
|
||||
|
||||
def train(self, maxiter=1000): # 训练数据
|
||||
for loop in range(maxiter): # 最大训练次数
|
||||
print("iter:%d" % loop)
|
||||
self._lastw = self._w[:]
|
||||
for i in range(self._n):
|
||||
ep = self._model_ep(i) # 计算第i个特征的模型期望
|
||||
self._w[i] += math.log(self._Ep_[i]/ep)/self._C # 更新参数
|
||||
print("w:", self._w)
|
||||
if self._convergence(): # 判断是否收敛
|
||||
break
|
||||
|
||||
|
||||
dataset = [['no', 'sunny', 'hot', 'high', 'FALSE'],
|
||||
['no', 'sunny', 'hot', 'high', 'TRUE'],
|
||||
['yes', 'overcast', 'hot', 'high', 'FALSE'],
|
||||
['yes', 'rainy', 'mild', 'high', 'FALSE'],
|
||||
['yes', 'rainy', 'cool', 'normal', 'FALSE'],
|
||||
['no', 'rainy', 'cool', 'normal', 'TRUE'],
|
||||
['yes', 'overcast', 'cool', 'normal', 'TRUE'],
|
||||
['no', 'sunny', 'mild', 'high', 'FALSE'],
|
||||
['yes', 'sunny', 'cool', 'normal', 'FALSE'],
|
||||
['yes', 'rainy', 'mild', 'normal', 'FALSE'],
|
||||
['yes', 'sunny', 'mild', 'normal', 'TRUE'],
|
||||
['yes', 'overcast', 'mild', 'high', 'TRUE'],
|
||||
['yes', 'overcast', 'hot', 'normal', 'FALSE'],
|
||||
['no', 'rainy', 'mild', 'high', 'TRUE']]
|
||||
|
||||
maxent = MaxEntropy()
|
||||
x = ['overcast', 'mild', 'high', 'FALSE']
|
||||
maxent.loadData(dataset)
|
||||
maxent.train()
|
||||
print('predict:', maxent.predict(x))
|
||||
File diff suppressed because one or more lines are too long
+461
File diff suppressed because one or more lines are too long
+257
@@ -0,0 +1,257 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"原文代码作者：https://github.com/wzyonggege/statistical-learning-method\n",
|
||||
"\n",
|
||||
"中文注释制作：机器学习初学者(微信公众号：ID:ai-start-com)\n",
|
||||
"\n",
|
||||
"配置环境：python 3.6\n",
|
||||
"\n",
|
||||
"代码全部测试通过。\n",
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 第9章 EM算法及其推广"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Expectation Maximization algorithm\n",
|
||||
"\n",
|
||||
"### Maximum likehood function\n",
|
||||
"\n",
|
||||
"[likehood & maximum likehood](http://fangs.in/post/thinkstats/likelihood/)\n",
|
||||
"\n",
|
||||
"> 在统计学中，似然函数（likelihood function，通常简写为likelihood，似然）是一个非常重要的内容，在非正式场合似然和概率（Probability）几乎是一对同义词，但是在统计学中似然和概率却是两个不同的概念。概率是在特定环境下某件事情发生的可能性，也就是结果没有产生之前依据环境所对应的参数来预测某件事情发生的可能性，比如抛硬币，抛之前我们不知道最后是哪一面朝上，但是根据硬币的性质我们可以推测任何一面朝上的可能性均为50%，这个概率只有在抛硬币之前才是有意义的，抛完硬币后的结果便是确定的；而似然刚好相反，是在确定的结果下去推测产生这个结果的可能环境（参数），还是抛硬币的例子，假设我们随机抛掷一枚硬币1,000次，结果500次人头朝上，500次数字朝上（实际情况一般不会这么理想，这里只是举个例子），我们很容易判断这是一枚标准的硬币，两面朝上的概率均为50%，这个过程就是我们运用出现的结果来判断这个事情本身的性质（参数），也就是似然。"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$P(Y|\\theta) = \\prod[\\pi p^{y_i}(1-p)^{1-y_i}+(1-\\pi) q^{y_i}(1-q)^{1-y_i}]$$\n",
|
||||
"\n",
|
||||
"### E step:\n",
|
||||
"\n",
|
||||
"$$\\mu^{i+1}=\\frac{\\pi (p^i)^{y_i}(1-(p^i))^{1-y_i}}{\\pi (p^i)^{y_i}(1-(p^i))^{1-y_i}+(1-\\pi) (q^i)^{y_i}(1-(q^i))^{1-y_i}}$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import math"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"pro_A, pro_B, por_C = 0.5, 0.5, 0.5\n",
|
||||
"\n",
|
||||
"def pmf(i, pro_A, pro_B, por_C):\n",
|
||||
" pro_1 = pro_A * math.pow(pro_B, data[i]) * math.pow((1-pro_B), 1-data[i])\n",
|
||||
" pro_2 = pro_A * math.pow(pro_C, data[i]) * math.pow((1-pro_C), 1-data[i])\n",
|
||||
" return pro_1 / (pro_1 + pro_2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### M step:\n",
|
||||
"\n",
|
||||
"$$\\pi^{i+1}=\\frac{1}{n}\\sum_{j=1}^n\\mu^{i+1}_j$$\n",
|
||||
"\n",
|
||||
"$$p^{i+1}=\\frac{\\sum_{j=1}^n\\mu^{i+1}_jy_i}{\\sum_{j=1}^n\\mu^{i+1}_j}$$\n",
|
||||
"\n",
|
||||
"$$q^{i+1}=\\frac{\\sum_{j=1}^n(1-\\mu^{i+1}_jy_i)}{\\sum_{j=1}^n(1-\\mu^{i+1}_j)}$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class EM:\n",
|
||||
" def __init__(self, prob):\n",
|
||||
" self.pro_A, self.pro_B, self.pro_C = prob\n",
|
||||
" \n",
|
||||
" # e_step\n",
|
||||
" def pmf(self, i):\n",
|
||||
" pro_1 = self.pro_A * math.pow(self.pro_B, data[i]) * math.pow((1-self.pro_B), 1-data[i])\n",
|
||||
" pro_2 = (1 - self.pro_A) * math.pow(self.pro_C, data[i]) * math.pow((1-self.pro_C), 1-data[i])\n",
|
||||
" return pro_1 / (pro_1 + pro_2)\n",
|
||||
" \n",
|
||||
" # m_step\n",
|
||||
" def fit(self, data):\n",
|
||||
" count = len(data)\n",
|
||||
" print('init prob:{}, {}, {}'.format(self.pro_A, self.pro_B, self.pro_C))\n",
|
||||
" for d in range(count):\n",
|
||||
" _ = yield\n",
|
||||
" _pmf = [self.pmf(k) for k in range(count)]\n",
|
||||
" pro_A = 1/ count * sum(_pmf)\n",
|
||||
" pro_B = sum([_pmf[k]*data[k] for k in range(count)]) / sum([_pmf[k] for k in range(count)])\n",
|
||||
" pro_C = sum([(1-_pmf[k])*data[k] for k in range(count)]) / sum([(1-_pmf[k]) for k in range(count)])\n",
|
||||
" print('{}/{} pro_a:{:.3f}, pro_b:{:.3f}, pro_c:{:.3f}'.format(d+1, count, pro_A, pro_B, pro_C))\n",
|
||||
" self.pro_A = pro_A\n",
|
||||
" self.pro_B = pro_B\n",
|
||||
" self.pro_C = pro_C\n",
|
||||
" "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"data=[1,1,0,1,0,0,1,0,1,1]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"init prob:0.5, 0.5, 0.5\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"em = EM(prob=[0.5, 0.5, 0.5])\n",
|
||||
"f = em.fit(data)\n",
|
||||
"next(f)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"1/10 pro_a:0.500, pro_b:0.600, pro_c:0.600\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 第一次迭代\n",
|
||||
"f.send(1)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"2/10 pro_a:0.500, pro_b:0.600, pro_c:0.600\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 第二次\n",
|
||||
"f.send(2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"init prob:0.4, 0.6, 0.7\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"em = EM(prob=[0.4, 0.6, 0.7])\n",
|
||||
"f2 = em.fit(data)\n",
|
||||
"next(f2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"1/10 pro_a:0.406, pro_b:0.537, pro_c:0.643\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"f2.send(1)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"2/10 pro_a:0.406, pro_b:0.537, pro_c:0.643\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"f2.send(2)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.2"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
Executable
BIN
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Executable
+37
@@ -0,0 +1,37 @@
|
||||
**《统计学习方法》简介**
|
||||
|
||||
**《统计学习方法》**，作者李航，本书全面系统地介绍了统计学习的主要方法，特别是监督学习方法，包括感知机、k近邻法、朴素贝叶斯法、决策树、逻辑斯谛回归与支持向量机、提升方法、EM算法、隐马尔可夫模型和条件随机场等。除第1章概论和最后一章总结外，每章介绍一种方法。叙述从具体问题或实例入手，由浅入深，阐明思路，给出必要的数学推导，便于读者掌握统计学习方法的实质，学会运用。
|
||||
|
||||
**目录:**
|
||||
|
||||
第1章 统计学习方法概论
|
||||
|
||||
第2章 感知机
|
||||
|
||||
第3章 k近邻法
|
||||
|
||||
第4章 朴素贝叶斯
|
||||
|
||||
第5章 决策树
|
||||
|
||||
第6章 逻辑斯谛回归
|
||||
|
||||
第7章 支持向量机
|
||||
|
||||
第8章 提升方法
|
||||
|
||||
第9章 EM算法及其推广
|
||||
|
||||
第10章 隐马尔可夫模型
|
||||
|
||||
第11章 条件随机场
|
||||
|
||||
第12章 统计学习方法总结
|
||||
|
||||
|
||||
|
||||
**《统计学习方法》课件**
|
||||
|
||||
作者袁春： 清华大学深圳研究生院，提供了全书12章的PPT课件。
|
||||
|
||||
整理：机器学习初学者 （微信公众号，ID:ai-start-com）
|
||||
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
+55
@@ -0,0 +1,55 @@
|
||||
《**统计学习方法**》可以说是机器学习的入门宝典，许多机器学习培训班、互联网企业的面试、笔试题目，很多都参考这本书。本站根据网上资料用**Python**复现了课程内容，并提供本书的代码实现、课件下载。
|
||||
|
||||
**《统计学习方法》**，作者李航，本书全面系统地介绍了统计学习的主要方法，特别是监督学习方法，包括感知机、k近邻法、朴素贝叶斯法、决策树、逻辑斯谛回归与支持向量机、提升方法、EM算法、隐马尔可夫模型和条件随机场等。除第1章概论和最后一章总结外，每章介绍一种方法。叙述从具体问题或实例入手，由浅入深，阐明思路，给出必要的数学推导，便于读者掌握统计学习方法的实质，学会运用。
|
||||
|
||||
**目录:**
|
||||
|
||||
第1章 统计学习方法概论
|
||||
|
||||
第2章 感知机
|
||||
|
||||
第3章 k近邻法
|
||||
|
||||
第4章 朴素贝叶斯
|
||||
|
||||
第5章 决策树
|
||||
|
||||
第6章 逻辑斯谛回归
|
||||
|
||||
第7章 支持向量机
|
||||
|
||||
第8章 提升方法
|
||||
|
||||
第9章 EM算法及其推广
|
||||
|
||||
第10章 隐马尔可夫模型
|
||||
|
||||
第11章 条件随机场
|
||||
|
||||
第12章 统计学习方法总结
|
||||
|
||||
**1.统计学习方法的代码实现（code文件夹）**
|
||||
|
||||
**《统计学习方法》**官方没有提供代码实现，但是网上有许多机器学习爱好者尝试对每一章的内容进行了代码实现。 本站在github网站搜集了一些代码进行整理，并作了一定的修改，使用**Python3.6**实现了第1-11章的课程代码。
|
||||
|
||||
**代码目录与截图：**
|
||||
|
||||
|
||||
|
||||
**2.《统计学习方法》课件（ppt文件夹）**
|
||||
|
||||
作者袁春： 清华大学深圳研究生院，提供了全书12章的PPT课件。
|
||||
|
||||
**参考**
|
||||
|
||||
[https://github.com/wzyonggege/statistical-learning-method](http://link.zhihu.com/?target=https%3A//github.com/wzyonggege/statistical-learning-method)
|
||||
|
||||
[https://github.com/WenDesi/lihang_book_algorithm](http://link.zhihu.com/?target=https%3A//github.com/WenDesi/lihang_book_algorithm)
|
||||
|
||||
[https://blog.csdn.net/tudaodiaozhale](http://link.zhihu.com/?target=https%3A//blog.csdn.net/tudaodiaozhale)
|
||||
|
||||
代码整理和修改：机器学习初学者 （微信公众号，ID:ai-start-com），qq群：774999266。
|
||||
|
||||
|
||||
|
||||
[我的知乎](https://www.zhihu.com/people/fengdu78)
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`Mikoto10032/DeepLearning`
|
||||
- 原始仓库：https://github.com/Mikoto10032/DeepLearning
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
BIN
Binary file not shown.
Executable
+93
@@ -0,0 +1,93 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
env/
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*,cover
|
||||
.hypothesis/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# IPython Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# celery beat schedule file
|
||||
celerybeat-schedule
|
||||
|
||||
# dotenv
|
||||
.env
|
||||
|
||||
# virtualenv
|
||||
venv/
|
||||
ENV/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
|
||||
# VSCode project settings
|
||||
.vscode/
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2018 ctgk
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
# PRML
|
||||
Python codes implementing algorithms described in Bishop's book "Pattern Recognition and Machine Learning"
|
||||
|
||||
## Required Packages
|
||||
- python 3
|
||||
- numpy
|
||||
- scipy
|
||||
- jupyter (optional: to run jupyter notebooks)
|
||||
- matplotlib (optional: to plot results in the notebooks)
|
||||
- sklearn (optional: to fetch data)
|
||||
|
||||
## Notebooks
|
||||
- [ch1. Introduction](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch01_Introduction.ipynb)
|
||||
- [ch2. Probability Distributions](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch02_Probability_Distributions.ipynb)
|
||||
- [ch3. Linear Models for Regression](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch03_Linear_Models_for_Regression.ipynb)
|
||||
- [ch4. Linear Models for Classification](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch04_Linear_Models_for_Classfication.ipynb)
|
||||
- [ch5. Neural Networks](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch05_Neural_Networks.ipynb)
|
||||
- [ch6. Kernel Methods](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch06_Kernel_Methods.ipynb)
|
||||
- [ch7. Sparse Kernel Machines](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch07_Sparse_Kernel_Machines.ipynb)
|
||||
- [ch9. Mixture Models and EM](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch09_Mixture_Models_and_EM.ipynb)
|
||||
- [ch10. Approximate Inference](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch10_Approximate_Inference.ipynb)
|
||||
- [ch11. Sampling Methods](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch11_Sampling_Methods.ipynb)
|
||||
- [ch12. Continuous Latent Variables](https://nbviewer.jupyter.org/github/ctgk/PRML/blob/master/notebooks/ch12_Continuous_Latent_Variables.ipynb)
|
||||
File diff suppressed because one or more lines are too long
+629
File diff suppressed because one or more lines are too long
+570
File diff suppressed because one or more lines are too long
+429
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,286 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 8. Graphical Models"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"import itertools\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import numpy as np\n",
|
||||
"from sklearn.datasets import fetch_mldata\n",
|
||||
"from prml import bayesnet as bn\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"np.random.seed(1234)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"b = bn.discrete([0.1, 0.9])\n",
|
||||
"f = bn.discrete([0.1, 0.9])\n",
|
||||
"\n",
|
||||
"g = bn.discrete([[[0.9, 0.8], [0.8, 0.2]], [[0.1, 0.2], [0.2, 0.8]]], b, f)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"b: DiscreteVariable(proba=[0.1 0.9])\n",
|
||||
"f: DiscreteVariable(proba=[0.1 0.9])\n",
|
||||
"g: DiscreteVariable(proba=[0.315 0.685])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(\"b:\", b)\n",
|
||||
"print(\"f:\", f)\n",
|
||||
"print(\"g:\", g)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"g.observe(0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"b: DiscreteVariable(proba=[0.25714286 0.74285714])\n",
|
||||
"f: DiscreteVariable(proba=[0.25714286 0.74285714])\n",
|
||||
"g: DiscreteVariable(observed=[1. 0.])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(\"b:\", b)\n",
|
||||
"print(\"f:\", f)\n",
|
||||
"print(\"g:\", g)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"b.observe(0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"b: DiscreteVariable(observed=[1. 0.])\n",
|
||||
"f: DiscreteVariable(proba=[0.11111111 0.88888889])\n",
|
||||
"g: DiscreteVariable(observed=[1. 0.])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(\"b:\", b)\n",
|
||||
"print(\"f:\", f)\n",
|
||||
"print(\"g:\", g)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 8.3.3 Illustration: Image de-noising"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<matplotlib.image.AxesImage at 0x2ada5ce4c18>"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAC0tJREFUeJzt3UGoZGeZxvH/M1E3MYsOIU0Tk4kjYTYu4tC4UYaehZJx03GRwaxaZtEuJqA7g5sERJBBndkJGWzsgTESiJomDBODOBNXIZ0gpmNPTJCe2KbpJvTCZCWadxb3tFw7996qW1WnTt1+/z8oqurcunXerttPfd853znnS1UhqZ+/mLoASdMw/FJThl9qyvBLTRl+qSnDLzVl+KWmDL/UlOGXmnrfOleWxMMJpZFVVeZ53VItf5L7krya5PUkDy/zXpLWK4se25/kJuBXwKeAi8ALwINV9cs9fseWXxrZOlr+jwOvV9Wvq+r3wPeB40u8n6Q1Wib8dwC/2fb84rDszyQ5meRskrNLrEvSii2zw2+nrsV7uvVV9RjwGNjtlzbJMi3/ReDObc8/BLy5XDmS1mWZ8L8A3JPkw0k+AHwOOLOasiSNbeFuf1X9IclDwDPATcCpqnplZZVJGtXCQ30Lrcxtfml0aznIR9LBZfilpgy/1JThl5oy/FJThl9qyvBLTRl+qSnDLzVl+KWmDL/UlOGXmjL8UlOGX2rK8EtNGX6pKcMvNWX4paYMv9SU4ZeaMvxSU2udolv9jHl16GSui9RqF7b8UlOGX2rK8EtNGX6pKcMvNWX4paYMv9TUUuP8SS4AbwN/BP5QVUdXUZQOjnXO8rzfdXscwN5WcZDP31XVWyt4H0lrZLdfamrZ8Bfw4yQvJjm5ioIkrcey3f5PVNWbSW4Hnk3yv1X13PYXDF8KfjFIGyar2mGT5FHgnar6xh6vmW7vkEYx5Q6/Wbru8Kuquf7hC3f7k9yc5JZrj4FPA+cWfT9J67VMt/8w8MPh2/V9wPeq6r9WUpWk0a2s2z/Xyuz2Hzib3K2fxW7/3hzqk5oy/FJThl9qyvBLTRl+qSnDLzXlpbub2+ShvFlDdbNq3+vnXYcBt7Pll5oy/FJThl9qyvBLTRl+qSnDLzVl+KWmHOe/AWzyWP2m8rLftvxSW4ZfasrwS00Zfqkpwy81Zfilpgy/1JTj/FrKsufcH9R13whs+aWmDL/UlOGXmjL8UlOGX2rK8EtNGX6pqZnhT3IqyZUk57YtuzXJs0leG+4PjVtmb1W1521MSfa8Lfv7y7y3ljNPy/9d4L7rlj0M/KSq7gF+MjyXdIDMDH9VPQdcvW7xceD08Pg0cP+K65I0skW3+Q9X1SWA4f721ZUkaR1GP7Y/yUng5NjrkbQ/i7b8l5McARjur+z2wqp6rKqOVtXRBdclaQSLhv8McGJ4fAJ4ajXlSFqXzHEJ48eBY8BtwGXgEeBHwBPAXcAbwANVdf1OwZ3ey3MsFzDlqakHechtmc/tgP+75yp+ZvhXyfDvzHCPY8prCUxp3vB7hJ/UlOGXmjL8UlOGX2rK8EtNGX6pKS/dvQbrOO1W2i9bfqkpwy81Zfilpgy/1JThl5oy/FJThl9qynH+A8BxfI3Bll9qyvBLTRl+qSnDLzVl+KWmDL/UlOGXmnKcfwWmvPS2tChbfqkpwy81Zfilpgy/1JThl5oy/FJThl9qamb4k5xKciXJuW3LHk3y2yQ/H26fGbfMG1uSPW/SGOZp+b8L3LfD8n+pqnuH23+utixJY5sZ/qp6Dri6hlokrdEy2/wPJfnFsFlwaGUVSVqLRcP/beAjwL3AJeCbu70wyckkZ5OcXXBdkkaQeU5KSXI38HRVfXQ/P9vhtTfkGTDLntjjTr1xjHnC1Sb/zapqruIWavmTHNn29LPAud1eK2kzzTylN8njwDHgtiQXgUeAY0nuBQq4AHxhxBoljWCubv/KVma3f0eb3IXcZF279bOM2u2XdPAZfqkpwy81Zfilpgy/1JThl5ry0t3aWA7ljcuWX2rK8EtNGX6pKcMvNWX4paYMv9SU4Zeacpxfkxn7dHLH8vdmyy81Zfilpgy/1JThl5oy/FJThl9qyvBLTTnOr1F5Tv7msuWXmjL8UlOGX2rK8EtNGX6pKcMvNWX4paZmhj/JnUl+muR8kleSfHFYfmuSZ5O8NtwfGr/cG1NV7Xmbct3L3paRZM+blpNZf6AkR4AjVfVSkluAF4H7gc8DV6vq60keBg5V1ZdnvNe4/5MncpAvSjF27csw4Iupqrk+uJktf1VdqqqXhsdvA+eBO4DjwOnhZafZ+kKQdEDsa5s/yd3Ax4DngcNVdQm2viCA21ddnKTxzH1sf5IPAk8CX6qq383bJUtyEji5WHmSxjJzmx8gyfuBp4Fnqupbw7JXgWNVdWnYL/DfVfXXM95nczcwl+A2/zjc5l/Myrb5s/UX+A5w/lrwB2eAE8PjE8BT+y1S0nTm2dv/SeBnwMvAu8Pir7C13f8EcBfwBvBAVV2d8V6b28wsYZNbz01myz6OeVv+ubr9q2L4tZ3hH8fKuv2SbkyGX2rK8EtNGX6pKcMvNWX4paa8dLeW4nDdwWXLLzVl+KWmDL/UlOGXmjL8UlOGX2rK8EtNOc6/ArPGujf5lF/H6fuy5ZeaMvxSU4ZfasrwS00Zfqkpwy81ZfilphznXwPH0rWJbPmlpgy/1JThl5oy/FJThl9qyvBLTRl+qamZ4U9yZ5KfJjmf5JUkXxyWP5rkt0l+Ptw+M365klYlsy40keQIcKSqXkpyC/AicD/wD8A7VfWNuVeWbO5VLaQbRFXNdVTZzCP8quoScGl4/HaS88Ady5UnaWr72uZPcjfwMeD5YdFDSX6R5FSSQ7v8zskkZ5OcXapSSSs1s9v/pxcmHwT+B/haVf0gyWHgLaCAr7K1afCPM97Dbr80snm7/XOFP8n7gaeBZ6rqWzv8/G7g6ar66Iz3MfzSyOYN/zx7+wN8Bzi/PfjDjsBrPguc22+RkqYzz97+TwI/A14G3h0WfwV4ELiXrW7/BeALw87Bvd7Lll8a2Uq7/ati+KXxrazbL+nGZPilpgy/1JThl5oy/FJThl9qyvBLTRl+qSnDLzVl+KWmDL/UlOGXmjL8UlOGX2pq3VN0vwX837bntw3LNtGm1rapdYG1LWqVtf3lvC9c6/n871l5craqjk5WwB42tbZNrQusbVFT1Wa3X2rK8EtNTR3+xyZe/142tbZNrQusbVGT1DbpNr+k6Uzd8kuayCThT3JfkleTvJ7k4Slq2E2SC0leHmYennSKsWEatCtJzm1bdmuSZ5O8NtzvOE3aRLVtxMzNe8wsPelnt2kzXq+925/kJuBXwKeAi8ALwINV9cu1FrKLJBeAo1U1+Zhwkr8F3gH+/dpsSEn+GbhaVV8fvjgPVdWXN6S2R9nnzM0j1bbbzNKfZ8LPbpUzXq/CFC3/x4HXq+rXVfV74PvA8Qnq2HhV9Rxw9brFx4HTw+PTbP3nWbtdatsIVXWpql4aHr8NXJtZetLPbo+6JjFF+O8AfrPt+UU2a8rvAn6c5MUkJ6cuZgeHr82MNNzfPnE915s5c/M6XTez9MZ8dovMeL1qU4R/p9lENmnI4RNV9TfA3wP/NHRvNZ9vAx9haxq3S8A3pyxmmFn6SeBLVfW7KWvZboe6Jvncpgj/ReDObc8/BLw5QR07qqo3h/srwA/Z2kzZJJevTZI63F+ZuJ4/qarLVfXHqnoX+Dcm/OyGmaWfBP6jqn4wLJ78s9uprqk+tynC/wJwT5IPJ/kA8DngzAR1vEeSm4cdMSS5Gfg0mzf78BngxPD4BPDUhLX8mU2ZuXm3maWZ+LPbtBmvJznIZxjK+FfgJuBUVX1t7UXsIMlfsdXaw9YZj9+bsrYkjwPH2Drr6zLwCPAj4AngLuAN4IGqWvuOt11qO8Y+Z24eqbbdZpZ+ngk/u1XOeL2SejzCT+rJI/ykpgy/1JThl5oy/FJThl9qyvBLTRl+qSnDLzX1//kg+AXbkzT0AAAAAElFTkSuQmCC\n",
|
||||
"text/plain": [
|
||||
"<Figure size 432x288 with 1 Axes>"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"needs_background": "light"
|
||||
},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"mnist = fetch_mldata(\"MNIST original\")\n",
|
||||
"x = mnist.data[0]\n",
|
||||
"binarized_img = (x > 127).astype(np.int).reshape(28, 28)\n",
|
||||
"plt.imshow(binarized_img, cmap=\"gray\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<matplotlib.image.AxesImage at 0x2ada5d84898>"
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAADMBJREFUeJzt3U+IpHedx/H3d6NeYg4JMbNDTHZcCV5yiNuNF2UZD0oUYeIhwZxGhG0PBtabIZcEFiEs6q4nYXYdHEGjgagZwrJRRI2nkJ6wmOj4J8hsnE0zo4xgchLNdw/9jLST7qrqep6nnuep7/sFTVfX1NTzrafqU8+vnu/z1C8yE0n1/M3QBUgahuGXijL8UlGGXyrK8EtFGX6pKMMvFWX4paIMv1TUm1a5sIhYy8MJNzY2Zv77uXPnBlt+38ueqqGfsz5lZixyu2hzeG9E3A18EbgO+M/MfHTO7dcy/PPWYcRCz0Uvy+972VM19HPWp97DHxHXAb8EPgBcBJ4D7s/Mn834P4Z/xcuf8ou4T0M/Z31aNPxtPvO/B3gpM3+dmX8EvgGcaHF/klaoTfhvBX6z5++LzXV/JSK2ImI7IrZbLEtSx9rs8NtvaPGGsVRmngJOwfoO+6UparPlvwjctufvtwOvtCtH0qq0Cf9zwB0R8Y6IeAvwMeBsN2VJ6tvSw/7M/FNEPAA8zW6r73Rm/rSzyiZk6D3Ds5a/znu126j6uPdq1ec/9ML8zL9yhr+eVbT6JE2Y4ZeKMvxSUYZfKsrwS0UZfqmolZ7PP6S2Lc0+W2J9tuNs5a2fWa+Xzc3Nhe/HLb9UlOGXijL8UlGGXyrK8EtFGX6pqDKtvnktr1We3Xgt23E6jK5eL275paIMv1SU4ZeKMvxSUYZfKsrwS0UZfqmoMn3+eda11z7lU5nb8ivNZ3PLLxVl+KWiDL9UlOGXijL8UlGGXyrK8EtFterzR8QF4FXgz8CfMnPx7w3uWN9921n33/a7AvrsKffdr27z2IZcbxX6+PN0cZDP+zPzdx3cj6QVctgvFdU2/Al8NyLORcRWFwVJWo22w/73ZuYrEXEL8L2I+HlmPrP3Bs2bgm8M0shEVydXRMQjwGuZ+bkZt+ntTA53+A2jz8e2zuutT5m50IpZetgfEddHxA1XLwMfBF5c9v4krVabYf8R4NvNu++bgK9n5n93UpWk3nU27F9oYXOG/W2GeWMeIo65tnmGnM+grTGv1z71PuyXNG2GXyrK8EtFGX6pKMMvFWX4paJG1eqrasiv1x5zK6/PadXXuQ1oq0/STIZfKsrwS0UZfqkowy8VZfilogy/VNTaTNE95dNm2/azx9yrH6spv1664pZfKsrwS0UZfqkowy8VZfilogy/VJThl4pamz7/OuvzvPa2hqxtzOtlCtzyS0UZfqkowy8VZfilogy/VJThl4oy/FJRc/v8EXEa+AhwOTPvbK67CfgmcAy4ANyXmb/vr8xds/q2Yz7/uu2542Pu44/1vsduDK/lRbb8XwHuvua6B4HvZ+YdwPebvyVNyNzwZ+YzwJVrrj4BnGkunwHu6bguST1b9jP/kczcAWh+39JdSZJWofdj+yNiC9jqezmSDmfZLf+liDgK0Py+fNANM/NUZm5m5uaSy5LUg2XDfxY42Vw+CTzZTTmSVmXuFN0R8RhwHLgZuAQ8DHwHeBy4HXgZuDczr90puN99tepZjaE9sozKrb4hn7MxT+Hd53pZdIruueHvUtvwr6u2z8GY3/j61OZNdZ3X+aLh9wg/qSjDLxVl+KWiDL9UlOGXijL8UlGT+uruqfb5K+vzORvz8RFT4JZfKsrwS0UZfqkowy8VZfilogy/VJThl4qaVJ9/qtqezz+ktrW3eWxTXm99mrVeNjcX/8Ist/xSUYZfKsrwS0UZfqkowy8VZfilogy/VNRK+/wbGxtsb28f+O9tesZD9oTbnjduP3t/Uz5fv8/ntKvXg1t+qSjDLxVl+KWiDL9UlOGXijL8UlGGXypq7hTdEXEa+AhwOTPvbK57BPgn4LfNzR7KzP+au7CiU3Tbxx9Gn8cBjPk563KK7q8Ad+9z/b9l5l3Nz9zgSxqXueHPzGeAKyuoRdIKtfnM/0BE/CQiTkfEjZ1VJGkllg3/l4B3AncBO8DnD7phRGxFxHZEHHxQv6SVm7vDDyAijgFPXd3ht+i/7XNbd/jtY8w7j6bMHX6zLbXlj4ije/78KPDiMvcjaThzT+mNiMeA48DNEXEReBg4HhF3AQlcAD7ZY42SerDQsL+zha3psL/tOhzzEHLK2jwvU35Oeh32S5o+wy8VZfilogy/VJThl4oy/FJRo5qi2yPhujfldVr1CL1VccsvFWX4paIMv1SU4ZeKMvxSUYZfKsrwS0WNqs8/1d7rmKeSHvM67Xu9TPWxr6put/xSUYZfKsrwS0UZfqkowy8VZfilogy/VNRK+/wbGxtsbx88a1ebfnnbXvuYe/V96vt8/3U9J7/tehvDMQhu+aWiDL9UlOGXijL8UlGGXyrK8EtFGX6pqLlTdEfEbcBXgb8FXgdOZeYXI+Im4JvAMeACcF9m/n7Ofa1ls3zK56WP+fiFKffxh7ToFN2LhP8ocDQzn4+IG4BzwD3Ax4ErmfloRDwI3JiZn5lzX+N9pbVg+Pth+JezaPjnDvszcyczn28uvwqcB24FTgBnmpudYfcNQdJEHOozf0QcA94NPAscycwd2H2DAG7pujhJ/Vn42P6IeCvwBPDpzPzDosOeiNgCtpYrT1Jf5n7mB4iINwNPAU9n5hea634BHM/MnWa/wA8z811z7me8HzBb8DN/P/zMv5zOPvPH7qP8MnD+avAbZ4GTzeWTwJOHLVLScBbZ2/8+4MfAC+y2+gAeYvdz/+PA7cDLwL2ZeWXOfbXazAz5dcdj3kJO1Zi3nlPWWauvS4Zfexn+fnQ27Je0ngy/VJThl4oy/FJRhl8qyvBLRY1qiu55bA2Nj8/JdLnll4oy/FJRhl8qyvBLRRl+qSjDLxVl+KWiJtXnH1KbfvaYTweu2qef8jf1dMUtv1SU4ZeKMvxSUYZfKsrwS0UZfqkowy8VZZ9/BcY880xVY+7jr+oYBLf8UlGGXyrK8EtFGX6pKMMvFWX4paIMv1TU3PBHxG0R8YOIOB8RP42If26ufyQi/i8i/qf5+XD/5fYnM2f+TFVEzPzR+KzqOYsFDig4ChzNzOcj4gbgHHAPcB/wWmZ+buGFRYw2RX65g9ZFZi70Yp17hF9m7gA7zeVXI+I8cGu78iQN7VCf+SPiGPBu4Nnmqgci4icRcToibjzg/2xFxHZEbLeqVFKn5g77/3LDiLcCPwI+m5nfiogjwO+ABP6F3Y8Gn5hzHw77pZ4tOuxfKPwR8WbgKeDpzPzCPv9+DHgqM++ccz+GX+rZouFfZG9/AF8Gzu8NfrMj8KqPAi8etkhJw1lkb//7gB8DLwCvN1c/BNwP3MXusP8C8Mlm5+Cs+5rsln+WtqMCRx3jM+XnpNNhf1cM/3LLHvMLbV1N+TnpbNgvaT0Zfqkowy8VZfilogy/VJThl4pa6Vd3b2xssL3dzyH+bVsvQ7Zu+mwV2oZczro+rr3c8ktFGX6pKMMvFWX4paIMv1SU4ZeKMvxSUas+pfe3wP/uuepmdr8KbIzGWttY6wJrW1aXtf1dZr5tkRuuNPxvWHjEdmZuDlbADGOtbax1gbUta6jaHPZLRRl+qaihw39q4OXPMtbaxloXWNuyBqlt0M/8koYz9JZf0kAGCX9E3B0Rv4iIlyLiwSFqOEhEXIiIF5qZhwedYqyZBu1yRLy457qbIuJ7EfGr5ve+06QNVNsoZm6eMbP0oOtubDNer3zYHxHXAb8EPgBcBJ4D7s/Mn620kANExAVgMzMH7wlHxD8CrwFfvTobUkT8K3AlMx9t3jhvzMzPjKS2RzjkzM091XbQzNIfZ8B11+WM110YYsv/HuClzPx1Zv4R+AZwYoA6Ri8znwGuXHP1CeBMc/kMuy+elTugtlHIzJ3MfL65/CpwdWbpQdfdjLoGMUT4bwV+s+fvi4xryu8EvhsR5yJia+hi9nHk6sxIze9bBq7nWnNnbl6la2aWHs26W2bG664NEf79vh9pTC2H92bmPwAfAj7VDG+1mC8B72R3Grcd4PNDFtPMLP0E8OnM/MOQtey1T12DrLchwn8RuG3P328HXhmgjn1l5ivN78vAt9n9mDIml65Oktr8vjxwPX+RmZcy88+Z+TrwHwy47pqZpZ8AvpaZ32quHnzd7VfXUOttiPA/B9wREe+IiLcAHwPODlDHG0TE9c2OGCLieuCDjG/24bPAyebySeDJAWv5K2OZufmgmaUZeN2NbcbrQQ7yaVoZ/w5cB5zOzM+uvIh9RMTfs7u1h91vNv76kLVFxGPAcXbP+roEPAx8B3gcuB14Gbg3M1e+4+2A2o5zyJmbe6rtoJmln2XAddfljNed1OMRflJNHuEnFWX4paIMv1SU4ZeKMvxSUYZfKsrwS0UZfqmo/wfVHcvvUKWKAwAAAABJRU5ErkJggg==\n",
|
||||
"text/plain": [
|
||||
"<Figure size 432x288 with 1 Axes>"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"needs_background": "light"
|
||||
},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"indices = np.random.choice(binarized_img.size, size=int(binarized_img.size * 0.1), replace=False)\n",
|
||||
"noisy_img = np.copy(binarized_img)\n",
|
||||
"noisy_img.ravel()[indices] = 1 - noisy_img.ravel()[indices]\n",
|
||||
"plt.imshow(noisy_img, cmap=\"gray\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"markov_random_field = np.array([\n",
|
||||
" [[bn.discrete([0.5, 0.5], name=f\"p(z_({i},{j}))\") for j in range(28)] for i in range(28)], \n",
|
||||
" [[bn.DiscreteVariable(2) for _ in range(28)] for _ in range(28)]])\n",
|
||||
"a = 0.9\n",
|
||||
"b = 0.9\n",
|
||||
"pa = [[a, 1 - a], [1 - a, a]]\n",
|
||||
"pb = [[b, 1 - b], [1 - b, b]]\n",
|
||||
"for i, j in itertools.product(range(28), range(28)):\n",
|
||||
" bn.discrete(pb, markov_random_field[0, i, j], out=markov_random_field[1, i, j], name=f\"p(x_({i},{j})|z_({i},{j}))\")\n",
|
||||
" if i != 27:\n",
|
||||
" bn.discrete(pa, out=[markov_random_field[0, i, j], markov_random_field[0, i + 1, j]], name=f\"p(z_({i},{j}), z_({i+1},{j}))\")\n",
|
||||
" if j != 27:\n",
|
||||
" bn.discrete(pa, out=[markov_random_field[0, i, j], markov_random_field[0, i, j + 1]], name=f\"p(z_({i},{j}), z_({i},{j+1}))\")\n",
|
||||
" markov_random_field[1, i, j].observe(noisy_img[i, j], proprange=0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<matplotlib.image.AxesImage at 0x2ada61c9f28>"
|
||||
]
|
||||
},
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAC4FJREFUeJzt3UHMZWV9x/Hvr6AbZAGhTCcIHWtINy6wEDeaZrrQUDfgAiOrMV2Mi5LUncQNJI2JadS2KxMaiWNSsSSoTEhTJKYtrggDMTJKUWJGHJnM1EwTYWWUfxfvGfI6vO9777z3nnvOO//vJ7m59z1z55z/e977u89zznPufVJVSOrnj6YuQNI0DL/UlOGXmjL8UlOGX2rK8EtNGX6pKcMvNWX4paau3eTGkng5oTSyqsoyz1up5U9yd5JXkrya5MFV1iVps7Lfa/uTXAP8FPgocBZ4Hri/qn6yx/+x5ZdGtomW/0PAq1X186r6LfAt4J4V1idpg1YJ/y3AL7f9fHZY9geSHE9yKsmpFbYlac1WOeG3U9fiHd36qnoEeATs9ktzskrLfxa4ddvP7wVeX60cSZuySvifB25P8r4k7wY+BZxcT1mSxrbvbn9V/S7JA8DTwDXAo1X147VVJmlU+x7q29fGPOaXRreRi3wkHVyGX2rK8EtNGX6pKcMvNWX4paY2+nn+OVs05JksNXoiHRi2/FJThl9qyvBLTRl+qSnDLzVl+KWmHOobOJSnbmz5paYMv9SU4ZeaMvxSU4ZfasrwS00Zfqkpx/k1W37Mely2/FJThl9qyvBLTRl+qSnDLzVl+KWmDL/U1Erj/EnOAG8Avwd+V1V3raMo9bDqDNFeB7CadVzk81dV9es1rEfSBtntl5paNfwFfC/JC0mOr6MgSZuxarf/w1X1epKbgWeS/E9VPbv9CcObgm8M0sxk1ZMub68oeRh4s6q+tMdz1rMxXRXW9drbTdcTflW11C++725/kuuSXH/pMfAx4PR+1ydps1bp9h8CvjO8u14LfLOq/mMtVUka3dq6/UttzG7/gbPJ18e62e3fm0N9UlOGX2rK8EtNGX6pKcMvNWX4pab86u4DYMyPrh7kobxF9vrdug4DbmfLLzVl+KWmDL/UlOGXmjL8UlOGX2rK8EtNOc5/Fbhax+oXjcWv8nv7td+2/FJbhl9qyvBLTRl+qSnDLzVl+KWmDL/UlOP8OrDGvA6gA1t+qSnDLzVl+KWmDL/UlOGXmjL8UlOGX2pqYfiTPJrkQpLT25bdmOSZJD8b7m8Yt8yrW1XteZtSksluGtcyLf/XgbsvW/Yg8P2quh34/vCzpANkYfir6lng4mWL7wFODI9PAPeuuS5JI9vvMf+hqjoHMNzfvL6SJG3C6Nf2JzkOHB97O5KuzH5b/vNJDgMM9xd2e2JVPVJVd1XVXfvclqQR7Df8J4Fjw+NjwJPrKUfSpmSJrzB+DDgK3AScBx4Cvgs8DtwGvAbcV1WXnxTcaV1+xnIHUw/n7eUgD7mtsl8P+O+9VPELw79Ohn9nq/4NDvILdUxjvrbnvM+XDb9X+ElNGX6pKcMvNWX4paYMv9SU4ZeaMvxSU4ZfasrwS00Zfqkpwy81Zfilpgy/1JThl5pyiu4DYM4fH53SnL8H4SCw5ZeaMvxSU4ZfasrwS00Zfqkpwy81Zfilphzn34Al5kbYUCVXl0X7zesA9mbLLzVl+KWmDL/UlOGXmjL8UlOGX2rK8EtNLQx/kkeTXEhyetuyh5P8KskPh9vHxy1z3qpqz5s0R8u0/F8H7t5h+T9W1R3D7d/XW5aksS0Mf1U9C1zcQC2SNmiVY/4HkvxoOCy4YW0VSdqI/Yb/q8D7gTuAc8CXd3tikuNJTiU5tc9tSRpBljkhleQI8FRVfeBK/m2H516VZ79WPannB3vGMebJ1jn/zapqqeL21fInObztx08Ap3d7rqR5WviR3iSPAUeBm5KcBR4Cjia5AyjgDPCZEWuUNIKluv1r25jd/h3NuQt5kK3ydznIf5NRu/2SDj7DLzVl+KWmDL/UlOGXmjL8UlN+dbdmq+sVeptiyy81Zfilpgy/1JThl5oy/FJThl9qyvBLTTnOr8mM/XFyx/L3ZssvNWX4paYMv9SU4ZeaMvxSU4ZfasrwS005zq9R+Zn8+bLll5oy/FJThl9qyvBLTRl+qSnDLzVl+KWmFo7zJ7kV+AbwJ8BbwCNV9c9JbgT+DTgCnAE+WVX/N16p0xpzvHrRusccz97kFO1XynH8cWWJF95h4HBVvZjkeuAF4F7g08DFqvpikgeBG6rqcwvWNd9X2gJThsTw60pU1VI7bmG3v6rOVdWLw+M3gJeBW4B7gBPD006w9YYg6YC4omP+JEeADwLPAYeq6hxsvUEAN6+7OEnjWfra/iTvAZ4APltVv1m2S5bkOHB8f+VJGsvCY36AJO8CngKerqqvDMteAY5W1bnhvMB/VdWfL1jPfA8wF/CYf/M85t+ftR3zZ+sv8DXg5UvBH5wEjg2PjwFPXmmRkqazzNn+jwA/AF5ia6gP4PNsHfc/DtwGvAbcV1UXF6xrvs3MAnNuIQ8qW/ZxLNvyL9XtXxfDr+0M/zjW1u2XdHUy/FJThl9qyvBLTRl+qSnDLzXlV3drJQ7XHVy2/FJThl9qyvBLTRl+qSnDLzVl+KWmDL/UlOP8S1plPHvOHwd2nL4vW36pKcMvNWX4paYMv9SU4ZeaMvxSU4Zfaspx/g1wLF1zZMsvNWX4paYMv9SU4ZeaMvxSU4ZfasrwS00tDH+SW5P8Z5KXk/w4yd8Nyx9O8qskPxxuHx+/XEnrkkVfNJHkMHC4ql5Mcj3wAnAv8Engzar60tIbS+b7rRbSVaKqlrqqbOEVflV1Djg3PH4jycvALauVJ2lqV3TMn+QI8EHguWHRA0l+lOTRJDfs8n+OJzmV5NRKlUpaq4Xd/refmLwH+G/gC1X17SSHgF8DBfw9W4cGf7NgHXb7pZEt2+1fKvxJ3gU8BTxdVV/Z4d+PAE9V1QcWrMfwSyNbNvzLnO0P8DXg5e3BH04EXvIJ4PSVFilpOsuc7f8I8APgJeCtYfHngfuBO9jq9p8BPjOcHNxrXbNt+ZfYDxuqRFrNWrv962L4pfGtrdsv6epk+KWmDL/UlOGXmjL8UlOGX2pqo+G/8847qapRbqtKsudNutrY8ktNGX6pKcMvNWX4paYMv9SU4ZeaMvxSU5v+SO//Ar/Ytugmtr4KbI7mWttc6wJr26911vanVfXHyzxxo+F/x8aTU1V112QF7GGutc21LrC2/ZqqNrv9UlOGX2pq6vA/MvH29zLX2uZaF1jbfk1S26TH/JKmM3XLL2kik4Q/yd1JXknyapIHp6hhN0nOJHlpmHl40inGhmnQLiQ5vW3ZjUmeSfKz4X7HadImqm0WMzfvMbP0pPtubjNeb7zbn+Qa4KfAR4GzwPPA/VX1k40WsoskZ4C7qmryMeEkfwm8CXzj0mxISf4BuFhVXxzeOG+oqs/NpLaHucKZm0eqbbeZpT/NhPtunTNer8MULf+HgFer6udV9VvgW8A9E9Qxe1X1LHDxssX3ACeGxyfYevFs3C61zUJVnauqF4fHbwCXZpaedN/tUdckpgj/LcAvt/18lnlN+V3A95K8kOT41MXs4NClmZGG+5snrudyC2du3qTLZpaezb7bz4zX6zZF+Hf6Tqw5DTl8uKr+Avhr4G+H7q2W81Xg/WxN43YO+PKUxQwzSz8BfLaqfjNlLdvtUNck+22K8J8Fbt3283uB1yeoY0dV9fpwfwH4DluHKXNy/tIkqcP9hYnreVtVna+q31fVW8C/MOG+G2aWfgL416r69rB48n23U11T7bcpwv88cHuS9yV5N/Ap4OQEdbxDkuuGEzEkuQ74GPObffgkcGx4fAx4csJa/sBcZm7ebWZpJt53c5vxepKLfIahjH8CrgEeraovbLyIHST5M7Zae4BrgW9OWVuSx4CjbH3q6zzwEPBd4HHgNuA14L6q2viJt11qO8oVztw8Um27zSz9HBPuu3XOeL2WerzCT+rJK/ykpgy/1JThl5oy/FJThl9qyvBLTRl+qSnDLzX1/2wNFDzYGLDhAAAAAElFTkSuQmCC\n",
|
||||
"text/plain": [
|
||||
"<Figure size 432x288 with 1 Axes>"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"needs_background": "light"
|
||||
},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"for _ in range(10000):\n",
|
||||
" i, j = np.random.choice(28, 2)\n",
|
||||
" markov_random_field[1, i, j].send_message(proprange=3)\n",
|
||||
"restored_img = np.zeros_like(noisy_img)\n",
|
||||
"for i, j in itertools.product(range(28), range(28)):\n",
|
||||
" restored_img[i, j] = np.argmax(markov_random_field[0, i, j].proba)\n",
|
||||
"plt.imshow(restored_img, cmap=\"gray\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+283
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+24
@@ -0,0 +1,24 @@
|
||||
from prml import (
|
||||
bayesnet,
|
||||
clustering,
|
||||
dimreduction,
|
||||
kernel,
|
||||
linear,
|
||||
markov,
|
||||
nn,
|
||||
rv,
|
||||
sampling
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"bayesnet",
|
||||
"clustering",
|
||||
"dimreduction",
|
||||
"kernel",
|
||||
"linear",
|
||||
"markov",
|
||||
"nn",
|
||||
"rv",
|
||||
"sampling"
|
||||
]
|
||||
@@ -0,0 +1,7 @@
|
||||
from prml.bayesnet.discrete import discrete, DiscreteVariable
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DiscreteVariable",
|
||||
"discrete"
|
||||
]
|
||||
+242
@@ -0,0 +1,242 @@
|
||||
import numpy as np
|
||||
from prml.bayesnet.probability_function import ProbabilityFunction
|
||||
from prml.bayesnet.random_variable import RandomVariable
|
||||
|
||||
|
||||
class DiscreteVariable(RandomVariable):
|
||||
"""
|
||||
Discrete random variable
|
||||
"""
|
||||
|
||||
def __init__(self, n_class:int):
|
||||
"""
|
||||
intialize a discrete random variable
|
||||
|
||||
parameters
|
||||
----------
|
||||
n_class : int
|
||||
number of classes
|
||||
|
||||
Attributes
|
||||
----------
|
||||
parent : DiscreteProbability, optional
|
||||
parent node this variable came out from
|
||||
message_from : dict
|
||||
dictionary of message from neighbor node and itself
|
||||
child : list of DiscreteProbability
|
||||
probability function this variable is conditioning
|
||||
proba : np.ndarray
|
||||
current estimate
|
||||
"""
|
||||
self.n_class = n_class
|
||||
self.parent = []
|
||||
self.message_from = {self: np.ones(n_class)}
|
||||
self.child = []
|
||||
self.is_observed = False
|
||||
|
||||
def __repr__(self):
|
||||
string = f"DiscreteVariable("
|
||||
if self.is_observed:
|
||||
string += f"observed={self.proba})"
|
||||
else:
|
||||
string += f"proba={self.proba})"
|
||||
return string
|
||||
|
||||
def add_parent(self, parent):
|
||||
self.parent.append(parent)
|
||||
|
||||
def add_child(self, child):
|
||||
self.child.append(child)
|
||||
self.message_from[child] = np.ones(self.n_class)
|
||||
|
||||
@property
|
||||
def proba(self):
|
||||
return self.posterior
|
||||
|
||||
def receive_message(self, message, giver, proprange):
|
||||
self.message_from[giver] = message
|
||||
self.summarize_message()
|
||||
self.send_message(proprange, exclude=giver)
|
||||
|
||||
def summarize_message(self):
|
||||
if self.is_observed:
|
||||
self.prior = self.message_from[self]
|
||||
self.likelihood = self.prior
|
||||
self.posterior = self.prior
|
||||
return
|
||||
|
||||
self.prior = np.ones(self.n_class)
|
||||
for func in self.parent:
|
||||
self.prior *= self.message_from[func]
|
||||
self.prior /= np.sum(self.prior, keepdims=True)
|
||||
|
||||
self.likelihood = np.copy(self.message_from[self])
|
||||
for func in self.child:
|
||||
self.likelihood *= self.message_from[func]
|
||||
|
||||
self.posterior = self.prior * self.likelihood
|
||||
self.posterior /= self.posterior.sum()
|
||||
|
||||
def send_message(self, proprange=-1, exclude=None):
|
||||
for func in self.parent:
|
||||
if func is not exclude:
|
||||
func.receive_message(self.likelihood, self, proprange)
|
||||
for func in self.child:
|
||||
if func is not exclude:
|
||||
func.receive_message(self.prior, self, proprange)
|
||||
|
||||
def observe(self, data:int, proprange=-1):
|
||||
"""
|
||||
set observed data of this variable
|
||||
|
||||
Parameters
|
||||
----------
|
||||
data : int
|
||||
observed data of this variable
|
||||
This must be smaller than n_class and must be non-negative
|
||||
propagate : int, optional
|
||||
Range to propagate the observation effect to the other random variable using belief propagation alg.
|
||||
If proprange=1, the effect only propagate to the neighboring random variables.
|
||||
Default is -1, which is infinite range.
|
||||
"""
|
||||
assert(0 <= data < self.n_class)
|
||||
self.is_observed = True
|
||||
self.receive_message(np.eye(self.n_class)[data], self, proprange=proprange)
|
||||
|
||||
|
||||
class DiscreteProbability(ProbabilityFunction):
|
||||
"""
|
||||
Discrete probability function
|
||||
"""
|
||||
|
||||
def __init__(self, table, *condition, out=None, name=None):
|
||||
"""
|
||||
initialize discrete probability function
|
||||
|
||||
Parameters
|
||||
----------
|
||||
table : (K, ...) np.ndarray or array-like
|
||||
probability table
|
||||
If a discrete variable A is conditioned with B and C,
|
||||
table[a,b,c] give probability of A=a when B=b and C=c.
|
||||
Thus, the sum along the first axis should equal to 1.
|
||||
If a table is 1 dimensional, the variable is not conditioned.
|
||||
condition : tuple of DiscreteVariable, optional
|
||||
parent node, discrete variable this function is conidtioned by
|
||||
len(condition) should equal to (table.ndim - 1)
|
||||
(Default is (), which means no condition)
|
||||
out : DiscreteVariable or list of DiscreteVariable, optional
|
||||
output of this discrete probability function
|
||||
Default is None which construct a new output instance
|
||||
name : str
|
||||
name of this discrete probability function
|
||||
"""
|
||||
self.table = np.asarray(table)
|
||||
self.condition = condition
|
||||
if condition:
|
||||
for var in condition:
|
||||
var.add_child(self)
|
||||
self.message_from = {var: var.prior for var in condition}
|
||||
|
||||
if out is None:
|
||||
self.out = [DiscreteVariable(len(table))]
|
||||
elif isinstance(out, DiscreteVariable):
|
||||
self.out = [out]
|
||||
else:
|
||||
self.out = out
|
||||
|
||||
for i, random_variable in enumerate(self.out):
|
||||
random_variable.add_parent(self)
|
||||
self.message_from[random_variable] = np.ones(np.size(self.table, i))
|
||||
|
||||
for random_variable in self.out:
|
||||
self.send_message_to(random_variable, proprange=0)
|
||||
|
||||
self.name = name
|
||||
|
||||
def __repr__(self):
|
||||
if self.name is not None:
|
||||
return self.name
|
||||
else:
|
||||
return super().__repr__()
|
||||
|
||||
def receive_message(self, message, giver, proprange):
|
||||
self.message_from[giver] = message
|
||||
if proprange:
|
||||
self.send_message(proprange, exclude=giver)
|
||||
|
||||
@staticmethod
|
||||
def expand_dims(x, ndim, axis):
|
||||
shape = [-1 if i == axis else 1 for i in range(ndim)]
|
||||
return x.reshape(*shape)
|
||||
|
||||
def compute_message_to(self, destination):
|
||||
proba = np.copy(self.table)
|
||||
for i, random_variable in enumerate(self.out):
|
||||
if random_variable is destination:
|
||||
index = i
|
||||
continue
|
||||
message = self.message_from[random_variable]
|
||||
proba *= self.expand_dims(message, proba.ndim, i)
|
||||
for i, random_variable in enumerate(self.condition, len(self.out)):
|
||||
if random_variable is destination:
|
||||
index = i
|
||||
continue
|
||||
message = self.message_from[random_variable]
|
||||
proba *= self.expand_dims(message, proba.ndim, i)
|
||||
axis = list(range(proba.ndim))
|
||||
axis.remove(index)
|
||||
message = np.sum(proba, axis=tuple(axis))
|
||||
message /= np.sum(message, keepdims=True)
|
||||
return message
|
||||
|
||||
def send_message_to(self, destination, proprange=-1):
|
||||
message = self.compute_message_to(destination)
|
||||
destination.receive_message(message, self, proprange)
|
||||
|
||||
def send_message(self, proprange, exclude=None):
|
||||
proprange = proprange - 1
|
||||
|
||||
for random_variable in self.out:
|
||||
if random_variable is not exclude:
|
||||
self.send_message_to(random_variable, proprange)
|
||||
|
||||
if proprange == 0: return
|
||||
|
||||
for random_variable in self.condition:
|
||||
if random_variable is not exclude:
|
||||
self.send_message_to(random_variable, proprange - 1)
|
||||
|
||||
|
||||
def discrete(table, *condition, out=None, name=None):
|
||||
"""
|
||||
discrete probability function
|
||||
|
||||
Parameters
|
||||
----------
|
||||
table : (K, ...) np.ndarray or array-like
|
||||
probability table
|
||||
If a discrete variable A is conditioned with B and C,
|
||||
table[a,b,c] give probability of A=a when B=b and C=c.
|
||||
Thus, the sum along the first axis should equal to 1.
|
||||
If a table is 1 dimensional, the variable is not conditioned.
|
||||
condition : tuple of DiscreteVariable, optional
|
||||
parent node, discrete variable this function is conidtioned by
|
||||
len(condition) should equal to (table.ndim - 1)
|
||||
(Default is (), which means no condition)
|
||||
out : DiscreteVariable, optional
|
||||
output of this discrete probability function
|
||||
Default is None which construct a new output instance
|
||||
name : str
|
||||
name of the discrete probability function
|
||||
|
||||
Returns
|
||||
-------
|
||||
DiscreteVariable
|
||||
output discrete random variable of discrete probability function
|
||||
"""
|
||||
function = DiscreteProbability(table, *condition, out=out, name=name)
|
||||
if len(function.out) == 1:
|
||||
return function.out[0]
|
||||
else:
|
||||
return function.out
|
||||
@@ -0,0 +1,2 @@
|
||||
class ProbabilityFunction(object):
|
||||
pass
|
||||
@@ -0,0 +1,4 @@
|
||||
class RandomVariable(object):
|
||||
"""
|
||||
Base class for random variable
|
||||
"""
|
||||
@@ -0,0 +1,6 @@
|
||||
from .k_means import KMeans
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KMeans"
|
||||
]
|
||||
@@ -0,0 +1,53 @@
|
||||
import numpy as np
|
||||
from scipy.spatial.distance import cdist
|
||||
|
||||
|
||||
class KMeans(object):
|
||||
|
||||
def __init__(self, n_clusters):
|
||||
self.n_clusters = n_clusters
|
||||
|
||||
def fit(self, X, iter_max=100):
|
||||
"""
|
||||
perform k-means algorithm
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input data
|
||||
iter_max : int
|
||||
maximum number of iterations
|
||||
|
||||
Returns
|
||||
-------
|
||||
centers : (n_clusters, n_features) ndarray
|
||||
center of each cluster
|
||||
"""
|
||||
I = np.eye(self.n_clusters)
|
||||
centers = X[np.random.choice(len(X), self.n_clusters, replace=False)]
|
||||
for _ in range(iter_max):
|
||||
prev_centers = np.copy(centers)
|
||||
D = cdist(X, centers)
|
||||
cluster_index = np.argmin(D, axis=1)
|
||||
cluster_index = I[cluster_index]
|
||||
centers = np.sum(X[:, None, :] * cluster_index[:, :, None], axis=0) / np.sum(cluster_index, axis=0)[:, None]
|
||||
if np.allclose(prev_centers, centers):
|
||||
break
|
||||
self.centers = centers
|
||||
|
||||
def predict(self, X):
|
||||
"""
|
||||
calculate closest cluster center index
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input data
|
||||
|
||||
Returns
|
||||
-------
|
||||
index : (sample_size,) ndarray
|
||||
indicates which cluster they belong
|
||||
"""
|
||||
D = cdist(X, self.centers)
|
||||
return np.argmin(D, axis=1)
|
||||
@@ -0,0 +1,10 @@
|
||||
from prml.dimreduction.autoencoder import Autoencoder
|
||||
from prml.dimreduction.bayesian_pca import BayesianPCA
|
||||
from prml.dimreduction.pca import PCA
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Autoencoder",
|
||||
"BayesianPCA",
|
||||
"PCA",
|
||||
]
|
||||
@@ -0,0 +1,38 @@
|
||||
import numpy as np
|
||||
from prml import nn
|
||||
|
||||
|
||||
class Autoencoder(nn.Network):
|
||||
|
||||
def __init__(self, *args):
|
||||
self.n_unit = len(args)
|
||||
super().__init__()
|
||||
for i in range(self.n_unit - 1):
|
||||
self.parameter[f"w_encode{i}"] = nn.Parameter(np.random.randn(args[i], args[i + 1]))
|
||||
self.parameter[f"b_encode{i}"] = nn.Parameter(np.zeros(args[i + 1]))
|
||||
self.parameter[f"w_decode{i}"] = nn.Parameter(np.random.randn(args[i + 1], args[i]))
|
||||
self.parameter[f"b_decode{i}"] = nn.Parameter(np.zeros(args[i]))
|
||||
|
||||
def transform(self, x):
|
||||
h = x
|
||||
for i in range(self.n_unit - 1):
|
||||
h = nn.tanh(h @ self.parameter[f"w_encode{i}"] + self.parameter[f"b_encode{i}"])
|
||||
return h.value
|
||||
|
||||
def forward(self, x):
|
||||
h = x
|
||||
for i in range(self.n_unit - 1):
|
||||
h = nn.tanh(h @ self.parameter[f"w_encode{i}"] + self.parameter[f"b_encode{i}"])
|
||||
for i in range(self.n_unit - 2, 0, -1):
|
||||
h = nn.tanh(h @ self.parameter[f"w_decode{i}"] + self.parameter[f"b_decode{i}"])
|
||||
x_ = h @ self.parameter["w_decode0"] + self.parameter["b_decode0"]
|
||||
self.px = nn.random.Gaussian(x_, 1., data=x)
|
||||
|
||||
def fit(self, x, n_iter=100, learning_rate=1e-3):
|
||||
optimizer = nn.optimizer.Adam(self.parameter, learning_rate)
|
||||
for _ in range(n_iter):
|
||||
self.clear()
|
||||
self.forward(x)
|
||||
log_likelihood = self.log_pdf()
|
||||
log_likelihood.backward()
|
||||
optimizer.update()
|
||||
@@ -0,0 +1,59 @@
|
||||
import numpy as np
|
||||
from prml.dimreduction.pca import PCA
|
||||
|
||||
|
||||
class BayesianPCA(PCA):
|
||||
|
||||
def fit(self, X, iter_max=100, initial="random"):
|
||||
"""
|
||||
empirical bayes estimation of pca parameters
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input data
|
||||
iter_max : int
|
||||
maximum number of em steps
|
||||
|
||||
Returns
|
||||
-------
|
||||
mean : (n_features,) ndarray
|
||||
sample mean fo the input data
|
||||
W : (n_features, n_components) ndarray
|
||||
projection matrix
|
||||
var : float
|
||||
variance of observation noise
|
||||
"""
|
||||
initial_list = ["random", "eigen"]
|
||||
self.mean = np.mean(X, axis=0)
|
||||
self.I = np.eye(self.n_components)
|
||||
if initial not in initial_list:
|
||||
print("availabel initializations are {}".format(initial_list))
|
||||
if initial == "random":
|
||||
self.W = np.eye(np.size(X, 1), self.n_components)
|
||||
self.var = 1.
|
||||
elif initial == "eigen":
|
||||
self.eigen(X)
|
||||
self.alpha = len(self.mean) / np.sum(self.W ** 2, axis=0).clip(min=1e-10)
|
||||
for i in range(iter_max):
|
||||
W = np.copy(self.W)
|
||||
stats = self._expectation(X - self.mean)
|
||||
self._maximization(X - self.mean, *stats)
|
||||
self.alpha = len(self.mean) / np.sum(self.W ** 2, axis=0).clip(min=1e-10)
|
||||
if np.allclose(W, self.W):
|
||||
break
|
||||
self.n_iter = i + 1
|
||||
|
||||
def _maximization(self, X, Ez, Ezz):
|
||||
self.W = X.T @ Ez @ np.linalg.inv(np.sum(Ezz, axis=0) + self.var * np.diag(self.alpha))
|
||||
self.var = np.mean(
|
||||
np.mean(X ** 2, axis=-1)
|
||||
- 2 * np.mean(Ez @ self.W.T * X, axis=-1)
|
||||
+ np.trace((Ezz @ self.W.T @ self.W).T) / len(self.mean))
|
||||
|
||||
def maximize(self, D, Ez, Ezz):
|
||||
self.W = D.T.dot(Ez).dot(np.linalg.inv(np.sum(Ezz, axis=0) + self.var * np.diag(self.alpha)))
|
||||
self.var = np.mean(
|
||||
np.mean(D ** 2, axis=-1)
|
||||
- 2 * np.mean(Ez.dot(self.W.T) * D, axis=-1)
|
||||
+ np.trace(Ezz.dot(self.W.T).dot(self.W).T) / self.ndim)
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class PCA(object):
|
||||
|
||||
def __init__(self, n_components):
|
||||
"""
|
||||
construct principal component analysis
|
||||
|
||||
Parameters
|
||||
----------
|
||||
n_components : int
|
||||
number of components
|
||||
"""
|
||||
assert isinstance(n_components, int)
|
||||
self.n_components = n_components
|
||||
|
||||
def fit(self, X, method="eigen", iter_max=100):
|
||||
"""
|
||||
maximum likelihood estimate of pca parameters
|
||||
x ~ \int_z N(x|Wz+mu,sigma^2)N(z|0,I)dz
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input data
|
||||
method : str
|
||||
method to estimate the parameters
|
||||
["eigen", "em"]
|
||||
iter_max : int
|
||||
maximum number of iterations for em algorithm
|
||||
|
||||
Attributes
|
||||
----------
|
||||
mean : (n_features,) ndarray
|
||||
sample mean of the data
|
||||
W : (n_features, n_components) ndarray
|
||||
projection matrix
|
||||
var : float
|
||||
variance of observation noise
|
||||
C : (n_features, n_features) ndarray
|
||||
variance of the marginal dist N(x|mean,C)
|
||||
Cinv : (n_features, n_features) ndarray
|
||||
precision of the marginal dist N(x|mean, C)
|
||||
"""
|
||||
method_list = ["eigen", "em"]
|
||||
if method not in method_list:
|
||||
print("availabel methods are {}".format(method_list))
|
||||
self.mean = np.mean(X, axis=0)
|
||||
getattr(self, method)(X - self.mean, iter_max)
|
||||
|
||||
def eigen(self, X, *arg):
|
||||
sample_size, n_features = X.shape
|
||||
if sample_size >= n_features:
|
||||
cov = np.cov(X, rowvar=False)
|
||||
values, vectors = np.linalg.eigh(cov)
|
||||
index = n_features - self.n_components
|
||||
else:
|
||||
cov = np.cov(X)
|
||||
values, vectors = np.linalg.eigh(cov)
|
||||
vectors = (X.T @ vectors) / np.sqrt(sample_size * values)
|
||||
index = sample_size - self.n_components
|
||||
self.I = np.eye(self.n_components)
|
||||
if index == 0:
|
||||
self.var = 0
|
||||
else:
|
||||
self.var = np.mean(values[:index])
|
||||
|
||||
self.W = vectors[:, index:].dot(np.sqrt(np.diag(values[index:]) - self.var * self.I))
|
||||
self.__M = self.W.T @ self.W + self.var * self.I
|
||||
self.C = self.W @ self.W.T + self.var * np.eye(n_features)
|
||||
if index == 0:
|
||||
self.Cinv = np.linalg.inv(self.C)
|
||||
else:
|
||||
self.Cinv = np.eye(n_features) / np.sqrt(self.var) - self.W @ np.linalg.inv(self.__M) @ self.W.T / self.var
|
||||
|
||||
def em(self, X, iter_max):
|
||||
self.I = np.eye(self.n_components)
|
||||
self.W = np.eye(np.size(X, 1), self.n_components)
|
||||
self.var = 1.
|
||||
for i in range(iter_max):
|
||||
W = np.copy(self.W)
|
||||
stats = self._expectation(X)
|
||||
self._maximization(X, *stats)
|
||||
if np.allclose(W, self.W):
|
||||
break
|
||||
self.C = self.W @ self.W.T + self.var * np.eye(np.size(X, 1))
|
||||
self.Cinv = np.linalg.inv(self.C)
|
||||
|
||||
def _expectation(self, X):
|
||||
self.__M = self.W.T @ self.W + self.var * self.I
|
||||
Minv = np.linalg.inv(self.__M)
|
||||
Ez = X @ self.W @ Minv
|
||||
Ezz = self.var * Minv + Ez[:, :, None] * Ez[:, None, :]
|
||||
return Ez, Ezz
|
||||
|
||||
def _maximization(self, X, Ez, Ezz):
|
||||
self.W = X.T @ Ez @ np.linalg.inv(np.sum(Ezz, axis=0))
|
||||
self.var = np.mean(
|
||||
np.mean(X ** 2, axis=1)
|
||||
- 2 * np.mean(Ez @ self.W.T * X, axis=1)
|
||||
+ np.trace((Ezz @ self.W.T @ self.W).T) / np.size(X, 1))
|
||||
|
||||
def transform(self, X):
|
||||
"""
|
||||
project input data into latent space
|
||||
p(Z|X) = N(Z|(X-mu)WMinv, sigma^-2M)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input data
|
||||
|
||||
Returns
|
||||
-------
|
||||
Z : (sample_size, n_components) ndarray
|
||||
projected input data
|
||||
"""
|
||||
return np.linalg.solve(self.__M, ((X - self.mean) @ self.W).T).T
|
||||
|
||||
def fit_transform(self, X, method="eigen"):
|
||||
"""
|
||||
perform pca and whiten the input data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input data
|
||||
|
||||
Returns
|
||||
-------
|
||||
Z : (sample_size, n_components) ndarray
|
||||
projected input data
|
||||
"""
|
||||
self.fit(X, method)
|
||||
return self.transform(X)
|
||||
|
||||
def proba(self, X):
|
||||
"""
|
||||
the marginal distribution of the observed variable
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input data
|
||||
|
||||
Returns
|
||||
-------
|
||||
p : (sample_size,) ndarray
|
||||
value of the marginal distribution
|
||||
"""
|
||||
d = X - self.mean
|
||||
return (
|
||||
np.exp(-0.5 * np.sum(d @ self.Cinv * d, axis=-1))
|
||||
/ np.sqrt(np.linalg.det(self.C))
|
||||
/ np.power(2 * np.pi, 0.5 * np.size(X, 1)))
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
from prml.kernel.polynomial import PolynomialKernel
|
||||
from prml.kernel.rbf import RBF
|
||||
|
||||
from prml.kernel.gaussian_process_classifier import GaussianProcessClassifier
|
||||
from prml.kernel.gaussian_process_regressor import GaussianProcessRegressor
|
||||
from prml.kernel.relevance_vector_classifier import RelevanceVectorClassifier
|
||||
from prml.kernel.relevance_vector_regressor import RelevanceVectorRegressor
|
||||
from prml.kernel.support_vector_classifier import SupportVectorClassifier
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PolynomialKernel",
|
||||
"RBF",
|
||||
"GaussianProcessClassifier",
|
||||
"GaussianProcessRegressor",
|
||||
"RelevanceVectorClassifier",
|
||||
"RelevanceVectorRegressor",
|
||||
"SupportVectorClassifier"
|
||||
]
|
||||
@@ -0,0 +1,37 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class GaussianProcessClassifier(object):
|
||||
|
||||
def __init__(self, kernel, noise_level=1e-4):
|
||||
"""
|
||||
construct gaussian process classifier
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kernel
|
||||
kernel function to be used to compute Gram matrix
|
||||
noise_level : float
|
||||
parameter to ensure the matrix to be positive
|
||||
"""
|
||||
self.kernel = kernel
|
||||
self.noise_level = noise_level
|
||||
|
||||
def _sigmoid(self, a):
|
||||
return np.tanh(a * 0.5) * 0.5 + 0.5
|
||||
|
||||
def fit(self, X, t):
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
self.X = X
|
||||
self.t = t
|
||||
Gram = self.kernel(X, X)
|
||||
self.covariance = Gram + np.eye(len(Gram)) * self.noise_level
|
||||
self.precision = np.linalg.inv(self.covariance)
|
||||
|
||||
def predict(self, X):
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
K = self.kernel(X, self.X)
|
||||
a_mean = K @ self.precision @ self.t
|
||||
return self._sigmoid(a_mean)
|
||||
@@ -0,0 +1,105 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class GaussianProcessRegressor(object):
|
||||
|
||||
def __init__(self, kernel, beta=1.):
|
||||
"""
|
||||
construct gaussian process regressor
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kernel
|
||||
kernel function
|
||||
beta : float
|
||||
precision parameter of observation noise
|
||||
"""
|
||||
self.kernel = kernel
|
||||
self.beta = beta
|
||||
|
||||
def fit(self, X, t, iter_max=0, learning_rate=0.1):
|
||||
"""
|
||||
maximum likelihood estimation of parameters in kernel function
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : ndarray (sample_size, n_features)
|
||||
input
|
||||
t : ndarray (sample_size,)
|
||||
corresponding target
|
||||
iter_max : int
|
||||
maximum number of iterations updating hyperparameters
|
||||
learning_rate : float
|
||||
updation coefficient
|
||||
|
||||
Attributes
|
||||
----------
|
||||
covariance : ndarray (sample_size, sample_size)
|
||||
variance covariance matrix of gaussian process
|
||||
precision : ndarray (sample_size, sample_size)
|
||||
precision matrix of gaussian process
|
||||
|
||||
Returns
|
||||
-------
|
||||
log_likelihood_list : list
|
||||
list of log likelihood value at each iteration
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
log_likelihood_list = [-np.Inf]
|
||||
self.X = X
|
||||
self.t = t
|
||||
I = np.eye(len(X))
|
||||
Gram = self.kernel(X, X)
|
||||
self.covariance = Gram + I / self.beta
|
||||
self.precision = np.linalg.inv(self.covariance)
|
||||
for i in range(iter_max):
|
||||
gradients = self.kernel.derivatives(X, X)
|
||||
updates = np.array(
|
||||
[-np.trace(self.precision.dot(grad)) + t.dot(self.precision.dot(grad).dot(self.precision).dot(t)) for grad in gradients])
|
||||
for j in range(iter_max):
|
||||
self.kernel.update_parameters(learning_rate * updates)
|
||||
Gram = self.kernel(X, X)
|
||||
self.covariance = Gram + I / self.beta
|
||||
self.precision = np.linalg.inv(self.covariance)
|
||||
log_like = self.log_likelihood()
|
||||
if log_like > log_likelihood_list[-1]:
|
||||
log_likelihood_list.append(log_like)
|
||||
break
|
||||
else:
|
||||
self.kernel.update_parameters(-learning_rate * updates)
|
||||
learning_rate *= 0.9
|
||||
log_likelihood_list.pop(0)
|
||||
return log_likelihood_list
|
||||
|
||||
def log_likelihood(self):
|
||||
return -0.5 * (
|
||||
np.linalg.slogdet(self.covariance)[1]
|
||||
+ self.t @ self.precision @ self.t
|
||||
+ len(self.t) * np.log(2 * np.pi))
|
||||
|
||||
def predict(self, X, with_error=False):
|
||||
"""
|
||||
mean of the gaussian process
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : ndarray (sample_size, n_features)
|
||||
input
|
||||
|
||||
Returns
|
||||
-------
|
||||
mean : ndarray (sample_size,)
|
||||
predictions of corresponding inputs
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
K = self.kernel(X, self.X)
|
||||
mean = K @ self.precision @ self.t
|
||||
if with_error:
|
||||
var = (
|
||||
self.kernel(X, X, False)
|
||||
+ 1 / self.beta
|
||||
- np.sum(K @ self.precision * K, axis=1))
|
||||
return mean.ravel(), np.sqrt(var.ravel())
|
||||
return mean
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class Kernel(object):
|
||||
"""
|
||||
Base class for kernel function
|
||||
"""
|
||||
|
||||
def _pairwise(self, x, y):
|
||||
"""
|
||||
all pairs of x and y
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : (sample_size, n_features)
|
||||
input
|
||||
y : (sample_size, n_features)
|
||||
another input
|
||||
|
||||
Returns
|
||||
-------
|
||||
output : tuple
|
||||
two array with shape (sample_size, sample_size, n_features)
|
||||
"""
|
||||
return (
|
||||
np.tile(x, (len(y), 1, 1)).transpose(1, 0, 2),
|
||||
np.tile(y, (len(x), 1, 1))
|
||||
)
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
import numpy as np
|
||||
from prml.kernel.kernel import Kernel
|
||||
|
||||
|
||||
class PolynomialKernel(Kernel):
|
||||
"""
|
||||
Polynomial kernel
|
||||
k(x,y) = (x @ y + c)^M
|
||||
"""
|
||||
|
||||
def __init__(self, degree=2, const=0.):
|
||||
"""
|
||||
construct Polynomial kernel
|
||||
|
||||
Parameters
|
||||
----------
|
||||
const : float
|
||||
a constant to be added
|
||||
degree : int
|
||||
degree of polynomial order
|
||||
"""
|
||||
self.const = const
|
||||
self.degree = degree
|
||||
|
||||
def __call__(self, x, y, pairwise=True):
|
||||
"""
|
||||
calculate pairwise polynomial kernel
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : (..., ndim) ndarray
|
||||
input
|
||||
y : (..., ndim) ndarray
|
||||
another input with the same shape
|
||||
|
||||
Returns
|
||||
-------
|
||||
output : ndarray
|
||||
polynomial kernel
|
||||
"""
|
||||
if pairwise:
|
||||
x, y = self._pairwise(x, y)
|
||||
return (np.sum(x * y, axis=-1) + self.const) ** self.degree
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
import numpy as np
|
||||
from prml.kernel.kernel import Kernel
|
||||
|
||||
|
||||
class RBF(Kernel):
|
||||
|
||||
def __init__(self, params):
|
||||
"""
|
||||
construct Radial basis kernel function
|
||||
|
||||
Parameters
|
||||
----------
|
||||
params : (ndim + 1,) ndarray
|
||||
parameters of radial basis function
|
||||
|
||||
Attributes
|
||||
----------
|
||||
ndim : int
|
||||
dimension of expected input data
|
||||
"""
|
||||
assert params.ndim == 1
|
||||
self.params = params
|
||||
self.ndim = len(params) - 1
|
||||
|
||||
def __call__(self, x, y, pairwise=True):
|
||||
"""
|
||||
calculate radial basis function
|
||||
k(x, y) = c0 * exp(-0.5 * c1 * (x1 - y1) ** 2 ...)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : ndarray [..., ndim]
|
||||
input of this kernel function
|
||||
y : ndarray [..., ndim]
|
||||
another input
|
||||
|
||||
Returns
|
||||
-------
|
||||
output : ndarray
|
||||
output of this radial basis function
|
||||
"""
|
||||
assert x.shape[-1] == self.ndim
|
||||
assert y.shape[-1] == self.ndim
|
||||
if pairwise:
|
||||
x, y = self._pairwise(x, y)
|
||||
d = self.params[1:] * (x - y) ** 2
|
||||
return self.params[0] * np.exp(-0.5 * np.sum(d, axis=-1))
|
||||
|
||||
def derivatives(self, x, y, pairwise=True):
|
||||
if pairwise:
|
||||
x, y = self._pairwise(x, y)
|
||||
d = self.params[1:] * (x - y) ** 2
|
||||
delta = np.exp(-0.5 * np.sum(d, axis=-1))
|
||||
deltas = -0.5 * (x - y) ** 2 * (delta * self.params[0])[:, :, None]
|
||||
return np.concatenate((np.expand_dims(delta, 0), deltas.T))
|
||||
|
||||
def update_parameters(self, updates):
|
||||
self.params += updates
|
||||
@@ -0,0 +1,122 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class RelevanceVectorClassifier(object):
|
||||
|
||||
def __init__(self, kernel, alpha=1.):
|
||||
"""
|
||||
construct relevance vector classifier
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kernel : Kernel
|
||||
kernel function to compute components of feature vectors
|
||||
alpha : float
|
||||
initial precision of prior weight distribution
|
||||
"""
|
||||
self.kernel = kernel
|
||||
self.alpha = alpha
|
||||
|
||||
def _sigmoid(self, a):
|
||||
return np.tanh(a * 0.5) * 0.5 + 0.5
|
||||
|
||||
def _map_estimate(self, X, t, w, n_iter=10):
|
||||
for _ in range(n_iter):
|
||||
y = self._sigmoid(X @ w)
|
||||
g = X.T @ (y - t) + self.alpha * w
|
||||
H = (X.T * y * (1 - y)) @ X + np.diag(self.alpha)
|
||||
w -= np.linalg.solve(H, g)
|
||||
return w, np.linalg.inv(H)
|
||||
|
||||
def fit(self, X, t, iter_max=100):
|
||||
"""
|
||||
maximize evidence with respect ot hyperparameter
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input
|
||||
t : (sample_size,) ndarray
|
||||
corresponding target
|
||||
iter_max : int
|
||||
maximum number of iterations
|
||||
|
||||
Attributes
|
||||
----------
|
||||
X : (N, n_features) ndarray
|
||||
relevance vector
|
||||
t : (N,) ndarray
|
||||
corresponding target
|
||||
alpha : (N,) ndarray
|
||||
hyperparameter for each weight or training sample
|
||||
cov : (N, N) ndarray
|
||||
covariance matrix of weight
|
||||
mean : (N,) ndarray
|
||||
mean of each weight
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
assert X.ndim == 2
|
||||
assert t.ndim == 1
|
||||
Phi = self.kernel(X, X)
|
||||
N = len(t)
|
||||
self.alpha = np.zeros(N) + self.alpha
|
||||
mean = np.zeros(N)
|
||||
for _ in range(iter_max):
|
||||
param = np.copy(self.alpha)
|
||||
mean, cov = self._map_estimate(Phi, t, mean, 10)
|
||||
gamma = 1 - self.alpha * np.diag(cov)
|
||||
self.alpha = gamma / np.square(mean)
|
||||
np.clip(self.alpha, 0, 1e10, out=self.alpha)
|
||||
if np.allclose(param, self.alpha):
|
||||
break
|
||||
mask = self.alpha < 1e8
|
||||
self.X = X[mask]
|
||||
self.t = t[mask]
|
||||
self.alpha = self.alpha[mask]
|
||||
Phi = self.kernel(self.X, self.X)
|
||||
mean = mean[mask]
|
||||
self.mean, self.covariance = self._map_estimate(Phi, self.t, mean, 100)
|
||||
|
||||
def predict(self, X):
|
||||
"""
|
||||
predict class label
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features)
|
||||
input
|
||||
|
||||
Returns
|
||||
-------
|
||||
label : (sample_size,) ndarray
|
||||
predicted label
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
assert X.ndim == 2
|
||||
phi = self.kernel(X, self.X)
|
||||
label = (phi @ self.mean > 0).astype(np.int)
|
||||
return label
|
||||
|
||||
def predict_proba(self, X):
|
||||
"""
|
||||
probability of input belonging class one
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input
|
||||
|
||||
Returns
|
||||
-------
|
||||
proba : (sample_size,) ndarray
|
||||
probability of predictive distribution p(C1|x)
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
assert X.ndim == 2
|
||||
phi = self.kernel(X, self.X)
|
||||
mu_a = phi @ self.mean
|
||||
var_a = np.sum(phi @ self.covariance * phi, axis=1)
|
||||
return self._sigmoid(mu_a / np.sqrt(1 + np.pi * var_a / 8))
|
||||
@@ -0,0 +1,102 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class RelevanceVectorRegressor(object):
|
||||
|
||||
def __init__(self, kernel, alpha=1., beta=1.):
|
||||
"""
|
||||
construct relevance vector regressor
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kernel : Kernel
|
||||
kernel function to compute components of feature vectors
|
||||
alpha : float
|
||||
initial precision of prior weight distribution
|
||||
beta : float
|
||||
precision of observation
|
||||
"""
|
||||
self.kernel = kernel
|
||||
self.alpha = alpha
|
||||
self.beta = beta
|
||||
|
||||
def fit(self, X, t, iter_max=1000):
|
||||
"""
|
||||
maximize evidence with respect to hyperparameter
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features) ndarray
|
||||
input
|
||||
t : (sample_size,) ndarray
|
||||
corresponding target
|
||||
iter_max : int
|
||||
maximum number of iterations
|
||||
|
||||
Attributes
|
||||
-------
|
||||
X : (N, n_features) ndarray
|
||||
relevance vector
|
||||
t : (N,) ndarray
|
||||
corresponding target
|
||||
alpha : (N,) ndarray
|
||||
hyperparameter for each weight or training sample
|
||||
cov : (N, N) ndarray
|
||||
covariance matrix of weight
|
||||
mean : (N,) ndarray
|
||||
mean of each weight
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
assert X.ndim == 2
|
||||
assert t.ndim == 1
|
||||
N = len(t)
|
||||
Phi = self.kernel(X, X)
|
||||
self.alpha = np.zeros(N) + self.alpha
|
||||
for _ in range(iter_max):
|
||||
params = np.hstack([self.alpha, self.beta])
|
||||
precision = np.diag(self.alpha) + self.beta * Phi.T @ Phi
|
||||
covariance = np.linalg.inv(precision)
|
||||
mean = self.beta * covariance @ Phi.T @ t
|
||||
gamma = 1 - self.alpha * np.diag(covariance)
|
||||
self.alpha = gamma / np.square(mean)
|
||||
np.clip(self.alpha, 0, 1e10, out=self.alpha)
|
||||
self.beta = (N - np.sum(gamma)) / np.sum((t - Phi.dot(mean)) ** 2)
|
||||
if np.allclose(params, np.hstack([self.alpha, self.beta])):
|
||||
break
|
||||
mask = self.alpha < 1e9
|
||||
self.X = X[mask]
|
||||
self.t = t[mask]
|
||||
self.alpha = self.alpha[mask]
|
||||
Phi = self.kernel(self.X, self.X)
|
||||
precision = np.diag(self.alpha) + self.beta * Phi.T @ Phi
|
||||
self.covariance = np.linalg.inv(precision)
|
||||
self.mean = self.beta * self.covariance @ Phi.T @ self.t
|
||||
|
||||
def predict(self, X, with_error=True):
|
||||
"""
|
||||
predict output with this model
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (sample_size, n_features)
|
||||
input
|
||||
with_error : bool
|
||||
if True, predict with standard deviation of the outputs
|
||||
|
||||
Returns
|
||||
-------
|
||||
mean : (sample_size,) ndarray
|
||||
mean of predictive distribution
|
||||
std : (sample_size,) ndarray
|
||||
standard deviation of predictive distribution
|
||||
"""
|
||||
if X.ndim == 1:
|
||||
X = X[:, None]
|
||||
assert X.ndim == 2
|
||||
phi = self.kernel(X, self.X)
|
||||
mean = phi @ self.mean
|
||||
if with_error:
|
||||
var = 1 / self.beta + np.sum(phi @ self.covariance * phi, axis=1)
|
||||
return mean, np.sqrt(var)
|
||||
return mean
|
||||
@@ -0,0 +1,107 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class SupportVectorClassifier(object):
|
||||
|
||||
def __init__(self, kernel, C=np.Inf):
|
||||
"""
|
||||
construct support vector classifier
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kernel : Kernel
|
||||
kernel function to compute inner products
|
||||
C : float
|
||||
penalty of misclassification
|
||||
"""
|
||||
self.kernel = kernel
|
||||
self.C = C
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, tol:float=1e-8):
|
||||
"""
|
||||
estimate support vectors and their parameters
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) np.ndarray
|
||||
training dependent variable
|
||||
binary -1 or 1
|
||||
tol : float, optional
|
||||
numerical tolerance (the default is 1e-8)
|
||||
"""
|
||||
|
||||
N = len(t)
|
||||
coef = np.zeros(N)
|
||||
grad = np.ones(N)
|
||||
Gram = self.kernel(X, X)
|
||||
|
||||
while True:
|
||||
tg = t * grad
|
||||
mask_up = (t == 1) & (coef < self.C - tol)
|
||||
mask_up |= (t == -1) & (coef > tol)
|
||||
mask_down = (t == -1) & (coef < self.C - tol)
|
||||
mask_down |= (t == 1) & (coef > tol)
|
||||
i = np.where(mask_up)[0][np.argmax(tg[mask_up])]
|
||||
j = np.where(mask_down)[0][np.argmin(tg[mask_down])]
|
||||
if tg[i] < tg[j] + tol:
|
||||
self.b = 0.5 * (tg[i] + tg[j])
|
||||
break
|
||||
else:
|
||||
A = self.C - coef[i] if t[i] == 1 else coef[i]
|
||||
B = coef[j] if t[j] == 1 else self.C - coef[j]
|
||||
direction = (tg[i] - tg[j]) / (Gram[i, i] - 2 * Gram[i, j] + Gram[j, j])
|
||||
direction = min(A, B, direction)
|
||||
coef[i] += direction * t[i]
|
||||
coef[j] -= direction * t[j]
|
||||
grad -= direction * t * (Gram[i] - Gram[j])
|
||||
support_mask = coef > tol
|
||||
self.a = coef[support_mask]
|
||||
self.X = X[support_mask]
|
||||
self.t = t[support_mask]
|
||||
|
||||
def lagrangian_function(self):
|
||||
return (
|
||||
np.sum(self.a)
|
||||
- self.a
|
||||
@ (self.t * self.t[:, None] * self.kernel(self.X, self.X))
|
||||
@ self.a)
|
||||
|
||||
def predict(self, x):
|
||||
"""
|
||||
predict labels of the input
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : (sample_size, n_features) ndarray
|
||||
input
|
||||
|
||||
Returns
|
||||
-------
|
||||
label : (sample_size,) ndarray
|
||||
predicted labels
|
||||
"""
|
||||
y = self.distance(x)
|
||||
label = np.sign(y)
|
||||
return label
|
||||
|
||||
def distance(self, x):
|
||||
"""
|
||||
calculate distance from the decision boundary
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : (sample_size, n_features) ndarray
|
||||
input
|
||||
|
||||
Returns
|
||||
-------
|
||||
distance : (sample_size,) ndarray
|
||||
distance from the boundary
|
||||
"""
|
||||
distance = np.sum(
|
||||
self.a * self.t
|
||||
* self.kernel(x, self.X),
|
||||
axis=-1) + self.b
|
||||
return distance
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
from prml.linear.bayesian_logistic_regression import BayesianLogisticRegression
|
||||
from prml.linear.bayesian_regression import BayesianRegression
|
||||
from prml.linear.emprical_bayes_regression import EmpiricalBayesRegression
|
||||
from prml.linear.least_squares_classifier import LeastSquaresClassifier
|
||||
from prml.linear.linear_regression import LinearRegression
|
||||
from prml.linear.fishers_linear_discriminant import FishersLinearDiscriminant
|
||||
from prml.linear.logistic_regression import LogisticRegression
|
||||
from prml.linear.perceptron import Perceptron
|
||||
from prml.linear.ridge_regression import RidgeRegression
|
||||
from prml.linear.softmax_regression import SoftmaxRegression
|
||||
from prml.linear.variational_linear_regression import VariationalLinearRegression
|
||||
from prml.linear.variational_logistic_regression import VariationalLogisticRegression
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BayesianLogisticRegression",
|
||||
"BayesianRegression",
|
||||
"EmpiricalBayesRegression",
|
||||
"LeastSquaresClassifier",
|
||||
"LinearRegression",
|
||||
"FishersLinearDiscriminant",
|
||||
"LogisticRegression",
|
||||
"Perceptron",
|
||||
"RidgeRegression",
|
||||
"SoftmaxRegression",
|
||||
"VariationalLinearRegression",
|
||||
"VariationalLogisticRegression"
|
||||
]
|
||||
@@ -0,0 +1,66 @@
|
||||
import numpy as np
|
||||
from prml.linear.logistic_regression import LogisticRegression
|
||||
|
||||
|
||||
class BayesianLogisticRegression(LogisticRegression):
|
||||
"""
|
||||
Logistic regression model
|
||||
|
||||
w ~ Gaussian(0, alpha^(-1)I)
|
||||
y = sigmoid(X @ w)
|
||||
t ~ Bernoulli(t|y)
|
||||
"""
|
||||
|
||||
def __init__(self, alpha:float=1.):
|
||||
self.alpha = alpha
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100):
|
||||
"""
|
||||
bayesian estimation of logistic regression model
|
||||
using Laplace approximation
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training data independent variable
|
||||
t : (N,) np.ndarray
|
||||
training data dependent variable
|
||||
binary 0 or 1
|
||||
max_iter : int, optional
|
||||
maximum number of paramter update iteration (the default is 100)
|
||||
"""
|
||||
w = np.zeros(np.size(X, 1))
|
||||
eye = np.eye(np.size(X, 1))
|
||||
self.w_mean = np.copy(w)
|
||||
self.w_precision = self.alpha * eye
|
||||
for _ in range(max_iter):
|
||||
w_prev = np.copy(w)
|
||||
y = self._sigmoid(X @ w)
|
||||
grad = X.T @ (y - t) + self.w_precision @ (w - self.w_mean)
|
||||
hessian = (X.T * y * (1 - y)) @ X + self.w_precision
|
||||
try:
|
||||
w -= np.linalg.solve(hessian, grad)
|
||||
except np.linalg.LinAlgError:
|
||||
break
|
||||
if np.allclose(w, w_prev):
|
||||
break
|
||||
self.w_mean = w
|
||||
self.w_precision = hessian
|
||||
|
||||
def proba(self, X:np.ndarray):
|
||||
"""
|
||||
compute probability of input belonging class 1
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training data independent variable
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
probability of positive
|
||||
"""
|
||||
mu_a = X @ self.w_mean
|
||||
var_a = np.sum(np.linalg.solve(self.w_precision, X.T).T * X, axis=1)
|
||||
return self._sigmoid(mu_a / np.sqrt(1 + np.pi * var_a / 8))
|
||||
@@ -0,0 +1,87 @@
|
||||
import numpy as np
|
||||
from prml.linear.regression import Regression
|
||||
|
||||
|
||||
class BayesianRegression(Regression):
|
||||
"""
|
||||
Bayesian regression model
|
||||
|
||||
w ~ N(w|0, alpha^(-1)I)
|
||||
y = X @ w
|
||||
t ~ N(t|X @ w, beta^(-1))
|
||||
"""
|
||||
|
||||
def __init__(self, alpha:float=1., beta:float=1.):
|
||||
self.alpha = alpha
|
||||
self.beta = beta
|
||||
self.w_mean = None
|
||||
self.w_precision = None
|
||||
|
||||
def _is_prior_defined(self) -> bool:
|
||||
return self.w_mean is not None and self.w_precision is not None
|
||||
|
||||
def _get_prior(self, ndim:int) -> tuple:
|
||||
if self._is_prior_defined():
|
||||
return self.w_mean, self.w_precision
|
||||
else:
|
||||
return np.zeros(ndim), self.alpha * np.eye(ndim)
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray):
|
||||
"""
|
||||
bayesian update of parameters given training dataset
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, n_features) np.ndarray
|
||||
training data independent variable
|
||||
t : (N,) np.ndarray
|
||||
training data dependent variable
|
||||
"""
|
||||
|
||||
mean_prev, precision_prev = self._get_prior(np.size(X, 1))
|
||||
|
||||
w_precision = precision_prev + self.beta * X.T @ X
|
||||
w_mean = np.linalg.solve(
|
||||
w_precision,
|
||||
precision_prev @ mean_prev + self.beta * X.T @ t
|
||||
)
|
||||
self.w_mean = w_mean
|
||||
self.w_precision = w_precision
|
||||
self.w_cov = np.linalg.inv(self.w_precision)
|
||||
|
||||
def predict(self, X:np.ndarray, return_std:bool=False, sample_size:int=None):
|
||||
"""
|
||||
return mean (and standard deviation) of predictive distribution
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, n_features) np.ndarray
|
||||
independent variable
|
||||
return_std : bool, optional
|
||||
flag to return standard deviation (the default is False)
|
||||
sample_size : int, optional
|
||||
number of samples to draw from the predictive distribution
|
||||
(the default is None, no sampling from the distribution)
|
||||
|
||||
Returns
|
||||
-------
|
||||
y : (N,) np.ndarray
|
||||
mean of the predictive distribution
|
||||
y_std : (N,) np.ndarray
|
||||
standard deviation of the predictive distribution
|
||||
y_sample : (N, sample_size) np.ndarray
|
||||
samples from the predictive distribution
|
||||
"""
|
||||
|
||||
if sample_size is not None:
|
||||
w_sample = np.random.multivariate_normal(
|
||||
self.w_mean, self.w_cov, size=sample_size
|
||||
)
|
||||
y_sample = X @ w_sample.T
|
||||
return y_sample
|
||||
y = X @ self.w_mean
|
||||
if return_std:
|
||||
y_var = 1 / self.beta + np.sum(X @ self.w_cov * X, axis=1)
|
||||
y_std = np.sqrt(y_var)
|
||||
return y, y_std
|
||||
return y
|
||||
@@ -0,0 +1,5 @@
|
||||
class Classifier(object):
|
||||
"""
|
||||
Base class for classifiers
|
||||
"""
|
||||
pass
|
||||
@@ -0,0 +1,86 @@
|
||||
import numpy as np
|
||||
from prml.linear.bayesian_regression import BayesianRegression
|
||||
|
||||
|
||||
class EmpiricalBayesRegression(BayesianRegression):
|
||||
"""
|
||||
Empirical Bayes Regression model
|
||||
a.k.a.
|
||||
type 2 maximum likelihood,
|
||||
generalized maximum likelihood,
|
||||
evidence approximation
|
||||
|
||||
w ~ N(w|0, alpha^(-1)I)
|
||||
y = X @ w
|
||||
t ~ N(t|X @ w, beta^(-1))
|
||||
evidence function p(t|X,alpha,beta) = S p(t|w;X,beta)p(w|0;alpha) dw
|
||||
"""
|
||||
|
||||
def __init__(self, alpha:float=1., beta:float=1.):
|
||||
super().__init__(alpha, beta)
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100):
|
||||
"""
|
||||
maximization of evidence function with respect to
|
||||
the hyperparameters alpha and beta given training dataset
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) np.ndarray
|
||||
training dependent variable
|
||||
max_iter : int
|
||||
maximum number of iteration
|
||||
"""
|
||||
M = X.T @ X
|
||||
eigenvalues = np.linalg.eigvalsh(M)
|
||||
eye = np.eye(np.size(X, 1))
|
||||
N = len(t)
|
||||
for _ in range(max_iter):
|
||||
params = [self.alpha, self.beta]
|
||||
|
||||
w_precision = self.alpha * eye + self.beta * X.T @ X
|
||||
w_mean = self.beta * np.linalg.solve(w_precision, X.T @ t)
|
||||
|
||||
gamma = np.sum(eigenvalues / (self.alpha + eigenvalues))
|
||||
self.alpha = float(gamma / np.sum(w_mean ** 2).clip(min=1e-10))
|
||||
self.beta = float(
|
||||
(N - gamma) / np.sum(np.square(t - X @ w_mean))
|
||||
)
|
||||
if np.allclose(params, [self.alpha, self.beta]):
|
||||
break
|
||||
self.w_mean = w_mean
|
||||
self.w_precision = w_precision
|
||||
self.w_cov = np.linalg.inv(w_precision)
|
||||
|
||||
def _log_prior(self, w):
|
||||
return -0.5 * self.alpha * np.sum(w ** 2)
|
||||
|
||||
def _log_likelihood(self, X, t, w):
|
||||
return -0.5 * self.beta * np.square(t - X @ w).sum()
|
||||
|
||||
def _log_posterior(self, X, t, w):
|
||||
return self._log_likelihood(X, t, w) + self._log_prior(w)
|
||||
|
||||
def log_evidence(self, X:np.ndarray, t:np.ndarray):
|
||||
"""
|
||||
logarithm or the evidence function
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
indenpendent variable
|
||||
t : (N,) np.ndarray
|
||||
dependent variable
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
log evidence
|
||||
"""
|
||||
N = len(t)
|
||||
D = np.size(X, 1)
|
||||
return 0.5 * (
|
||||
D * np.log(self.alpha) + N * np.log(self.beta)
|
||||
- np.linalg.slogdet(self.w_precision)[1] - D * np.log(2 * np.pi)
|
||||
) + self._log_posterior(X, t, self.w_mean)
|
||||
@@ -0,0 +1,80 @@
|
||||
import numpy as np
|
||||
from prml.linear.classifier import Classifier
|
||||
from prml.rv.gaussian import Gaussian
|
||||
|
||||
|
||||
class FishersLinearDiscriminant(Classifier):
|
||||
"""
|
||||
Fisher's Linear discriminant model
|
||||
"""
|
||||
|
||||
def __init__(self, w:np.ndarray=None, threshold:float=None):
|
||||
self.w = w
|
||||
self.threshold = threshold
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray):
|
||||
"""
|
||||
estimate parameter given training dataset
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training dataset independent variable
|
||||
t : (N,) np.ndarray
|
||||
training dataset dependent variable
|
||||
binary 0 or 1
|
||||
"""
|
||||
X0 = X[t == 0]
|
||||
X1 = X[t == 1]
|
||||
m0 = np.mean(X0, axis=0)
|
||||
m1 = np.mean(X1, axis=0)
|
||||
cov_inclass = np.cov(X0, rowvar=False) + np.cov(X1, rowvar=False)
|
||||
self.w = np.linalg.solve(cov_inclass, m1 - m0)
|
||||
self.w /= np.linalg.norm(self.w).clip(min=1e-10)
|
||||
|
||||
g0 = Gaussian()
|
||||
g0.fit((X0 @ self.w))
|
||||
g1 = Gaussian()
|
||||
g1.fit((X1 @ self.w))
|
||||
root = np.roots([
|
||||
g1.var - g0.var,
|
||||
2 * (g0.var * g1.mu - g1.var * g0.mu),
|
||||
g1.var * g0.mu ** 2 - g0.var * g1.mu ** 2
|
||||
- g1.var * g0.var * np.log(g1.var / g0.var)
|
||||
])
|
||||
if g0.mu < root[0] < g1.mu or g1.mu < root[0] < g0.mu:
|
||||
self.threshold = root[0]
|
||||
else:
|
||||
self.threshold = root[1]
|
||||
|
||||
def transform(self, X:np.ndarray):
|
||||
"""
|
||||
project data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable
|
||||
|
||||
Returns
|
||||
-------
|
||||
y : (N,) np.ndarray
|
||||
projected data
|
||||
"""
|
||||
return X @ self.w
|
||||
|
||||
def classify(self, X:np.ndarray):
|
||||
"""
|
||||
classify input data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable to be classified
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
binary class for each input
|
||||
"""
|
||||
return (X @ self.w > self.threshold).astype(np.int)
|
||||
@@ -0,0 +1,48 @@
|
||||
import numpy as np
|
||||
from prml.linear.classifier import Classifier
|
||||
from prml.preprocess.label_transformer import LabelTransformer
|
||||
|
||||
|
||||
class LeastSquaresClassifier(Classifier):
|
||||
"""
|
||||
Least squares classifier model
|
||||
|
||||
X : (N, D)
|
||||
W : (D, K)
|
||||
y = argmax_k X @ W
|
||||
"""
|
||||
|
||||
def __init__(self, W:np.ndarray=None):
|
||||
self.W = W
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray):
|
||||
"""
|
||||
least squares fitting for classification
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) or (N, K) np.ndarray
|
||||
training dependent variable
|
||||
in class index (N,) or one-of-k coding (N,K)
|
||||
"""
|
||||
if t.ndim == 1:
|
||||
t = LabelTransformer().encode(t)
|
||||
self.W = np.linalg.pinv(X) @ t
|
||||
|
||||
def classify(self, X:np.ndarray):
|
||||
"""
|
||||
classify input data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable to be classified
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
class index for each input
|
||||
"""
|
||||
return np.argmax(X @ self.W, axis=-1)
|
||||
@@ -0,0 +1,48 @@
|
||||
import numpy as np
|
||||
from prml.linear.regression import Regression
|
||||
|
||||
|
||||
class LinearRegression(Regression):
|
||||
"""
|
||||
Linear regression model
|
||||
y = X @ w
|
||||
t ~ N(t|X @ w, var)
|
||||
"""
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray):
|
||||
"""
|
||||
perform least squares fitting
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) np.ndarray
|
||||
training dependent variable
|
||||
"""
|
||||
self.w = np.linalg.pinv(X) @ t
|
||||
self.var = np.mean(np.square(X @ self.w - t))
|
||||
|
||||
def predict(self, X:np.ndarray, return_std:bool=False):
|
||||
"""
|
||||
make prediction given input
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
samples to predict their output
|
||||
return_std : bool, optional
|
||||
returns standard deviation of each predition if True
|
||||
|
||||
Returns
|
||||
-------
|
||||
y : (N,) np.ndarray
|
||||
prediction of each sample
|
||||
y_std : (N,) np.ndarray
|
||||
standard deviation of each predition
|
||||
"""
|
||||
y = X @ self.w
|
||||
if return_std:
|
||||
y_std = np.sqrt(self.var) + np.zeros_like(y)
|
||||
return y, y_std
|
||||
return y
|
||||
@@ -0,0 +1,77 @@
|
||||
import numpy as np
|
||||
from prml.linear.classifier import Classifier
|
||||
|
||||
|
||||
class LogisticRegression(Classifier):
|
||||
"""
|
||||
Logistic regression model
|
||||
|
||||
y = sigmoid(X @ w)
|
||||
t ~ Bernoulli(t|y)
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _sigmoid(a):
|
||||
return np.tanh(a * 0.5) * 0.5 + 0.5
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100):
|
||||
"""
|
||||
maximum likelihood estimation of logistic regression model
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training data independent variable
|
||||
t : (N,) np.ndarray
|
||||
training data dependent variable
|
||||
binary 0 or 1
|
||||
max_iter : int, optional
|
||||
maximum number of paramter update iteration (the default is 100)
|
||||
"""
|
||||
w = np.zeros(np.size(X, 1))
|
||||
for _ in range(max_iter):
|
||||
w_prev = np.copy(w)
|
||||
y = self._sigmoid(X @ w)
|
||||
grad = X.T @ (y - t)
|
||||
hessian = (X.T * y * (1 - y)) @ X
|
||||
try:
|
||||
w -= np.linalg.solve(hessian, grad)
|
||||
except np.linalg.LinAlgError:
|
||||
break
|
||||
if np.allclose(w, w_prev):
|
||||
break
|
||||
self.w = w
|
||||
|
||||
def proba(self, X:np.ndarray):
|
||||
"""
|
||||
compute probability of input belonging class 1
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training data independent variable
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
probability of positive
|
||||
"""
|
||||
return self._sigmoid(X @ self.w)
|
||||
|
||||
def classify(self, X:np.ndarray, threshold:float=0.5):
|
||||
"""
|
||||
classify input data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable to be classified
|
||||
threshold : float, optional
|
||||
threshold of binary classification (default is 0.5)
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
binary class for each input
|
||||
"""
|
||||
return (self.proba(X) > threshold).astype(np.int)
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
import numpy as np
|
||||
from prml.linear.classifier import Classifier
|
||||
|
||||
|
||||
class Perceptron(Classifier):
|
||||
"""
|
||||
Perceptron model
|
||||
"""
|
||||
|
||||
def fit(self, X, t, max_epoch=100):
|
||||
"""
|
||||
fit perceptron model on given input pair
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,)
|
||||
training dependent variable
|
||||
binary -1 or 1
|
||||
max_epoch : int, optional
|
||||
maximum number of epoch (the default is 100)
|
||||
"""
|
||||
self.w = np.zeros(np.size(X, 1))
|
||||
for _ in range(max_epoch):
|
||||
N = len(t)
|
||||
index = np.random.permutation(N)
|
||||
X = X[index]
|
||||
t = t[index]
|
||||
for x, label in zip(X, t):
|
||||
self.w += x * label
|
||||
if (X @ self.w * t > 0).all():
|
||||
break
|
||||
else:
|
||||
continue
|
||||
break
|
||||
|
||||
def classify(self, X):
|
||||
"""
|
||||
classify input data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable to be classified
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
binary class (-1 or 1) for each input
|
||||
"""
|
||||
return np.sign(X @ self.w).astype(np.int)
|
||||
@@ -0,0 +1,5 @@
|
||||
class Regression(object):
|
||||
"""
|
||||
Base class for regressors
|
||||
"""
|
||||
pass
|
||||
@@ -0,0 +1,44 @@
|
||||
import numpy as np
|
||||
from prml.linear.regression import Regression
|
||||
|
||||
|
||||
class RidgeRegression(Regression):
|
||||
"""
|
||||
Ridge regression model
|
||||
|
||||
w* = argmin |t - X @ w| + alpha * |w|_2^2
|
||||
"""
|
||||
|
||||
def __init__(self, alpha:float=1.):
|
||||
self.alpha = alpha
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray):
|
||||
"""
|
||||
maximum a posteriori estimation of parameter
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training data independent variable
|
||||
t : (N,) np.ndarray
|
||||
training data dependent variable
|
||||
"""
|
||||
|
||||
eye = np.eye(np.size(X, 1))
|
||||
self.w = np.linalg.solve(self.alpha * eye + X.T @ X, X.T @ t)
|
||||
|
||||
def predict(self, X:np.ndarray):
|
||||
"""
|
||||
make prediction given input
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
samples to predict their output
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
prediction of each input
|
||||
"""
|
||||
return X @ self.w
|
||||
@@ -0,0 +1,83 @@
|
||||
import numpy as np
|
||||
from prml.linear.classifier import Classifier
|
||||
from prml.preprocess.label_transformer import LabelTransformer
|
||||
|
||||
|
||||
class SoftmaxRegression(Classifier):
|
||||
"""
|
||||
Softmax regression model
|
||||
aka
|
||||
multinomial logistic regression,
|
||||
multiclass logistic regression,
|
||||
maximum entropy classifier.
|
||||
|
||||
y = softmax(X @ W)
|
||||
t ~ Categorical(t|y)
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _softmax(a):
|
||||
a_max = np.max(a, axis=-1, keepdims=True)
|
||||
exp_a = np.exp(a - a_max)
|
||||
return exp_a / np.sum(exp_a, axis=-1, keepdims=True)
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, max_iter:int=100, learning_rate:float=0.1):
|
||||
"""
|
||||
maximum likelihood estimation of the parameter
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) or (N, K) np.ndarray
|
||||
training dependent variable
|
||||
in class index or one-of-k encoding
|
||||
max_iter : int, optional
|
||||
maximum number of iteration (the default is 100)
|
||||
learning_rate : float, optional
|
||||
learning rate of gradient descent (the default is 0.1)
|
||||
"""
|
||||
if t.ndim == 1:
|
||||
t = LabelTransformer().encode(t)
|
||||
self.n_classes = np.size(t, 1)
|
||||
W = np.zeros((np.size(X, 1), self.n_classes))
|
||||
for _ in range(max_iter):
|
||||
W_prev = np.copy(W)
|
||||
y = self._softmax(X @ W)
|
||||
grad = X.T @ (y - t)
|
||||
W -= learning_rate * grad
|
||||
if np.allclose(W, W_prev):
|
||||
break
|
||||
self.W = W
|
||||
|
||||
def proba(self, X:np.ndarray):
|
||||
"""
|
||||
compute probability of input belonging each class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N, K) np.ndarray
|
||||
probability of each class
|
||||
"""
|
||||
return self._softmax(X @ self.W)
|
||||
|
||||
def classify(self, X:np.ndarray):
|
||||
"""
|
||||
classify input data
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable to be classified
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
class index for each input
|
||||
"""
|
||||
return np.argmax(self.proba(X), axis=-1)
|
||||
@@ -0,0 +1,93 @@
|
||||
import numpy as np
|
||||
from prml.linear.regression import Regression
|
||||
|
||||
|
||||
class VariationalLinearRegression(Regression):
|
||||
"""
|
||||
variational bayesian estimation of linear regression model
|
||||
p(w,alpha|X,t)
|
||||
~ q(w)q(alpha)
|
||||
= N(w|w_mean, w_var)Gamma(alpha|a,b)
|
||||
|
||||
Attributes
|
||||
----------
|
||||
a : float
|
||||
a parameter of variational posterior gamma distribution
|
||||
b : float
|
||||
another parameter of variational posterior gamma distribution
|
||||
w_mean : (n_features,) ndarray
|
||||
mean of variational posterior gaussian distribution
|
||||
w_var : (n_features, n_feautures) ndarray
|
||||
variance of variational posterior gaussian distribution
|
||||
n_iter : int
|
||||
number of iterations performed
|
||||
"""
|
||||
|
||||
def __init__(self, beta:float=1., a0:float=1., b0:float=1.):
|
||||
"""
|
||||
construct variational linear regressor
|
||||
Parameters
|
||||
----------
|
||||
beta : float
|
||||
precision of observation noise
|
||||
a0 : float
|
||||
a parameter of prior gamma distribution
|
||||
Gamma(alpha|a0,b0)
|
||||
b0 : float
|
||||
another parameter of prior gamma distribution
|
||||
Gamma(alpha|a0,b0)
|
||||
"""
|
||||
self.beta = beta
|
||||
self.a0 = a0
|
||||
self.b0 = b0
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, iter_max:int=100):
|
||||
"""
|
||||
variational bayesian estimation of parameter
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) np.ndarray
|
||||
training dependent variable
|
||||
iter_max : int, optional
|
||||
maximum number of iteration (the default is 100)
|
||||
"""
|
||||
D = np.size(X, 1)
|
||||
self.a = self.a0 + 0.5 * D
|
||||
self.b = self.b0
|
||||
I = np.eye(D)
|
||||
for _ in range(iter_max):
|
||||
param = self.b
|
||||
self.w_var = np.linalg.inv(self.a * I / self.b + self.beta * X.T @ X)
|
||||
self.w_mean = self.beta * self.w_var @ X.T @ t
|
||||
self.b = self.b0 + 0.5 * (np.sum(self.w_mean ** 2) + np.trace(self.w_var))
|
||||
if np.allclose(self.b, param):
|
||||
break
|
||||
|
||||
def predict(self, X:np.ndarray, return_std:bool=False):
|
||||
"""
|
||||
make prediction of input
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
independent variable
|
||||
return_std : bool, optional
|
||||
return standard deviation of predictive distribution if True
|
||||
(the default is False)
|
||||
|
||||
Returns
|
||||
-------
|
||||
y : (N,) np.ndarray
|
||||
mean of predictive distribution
|
||||
y_std : (N,) np.ndarray
|
||||
standard deviation of predictive distribution
|
||||
"""
|
||||
y = X @ self.w_mean
|
||||
if return_std:
|
||||
y_var = 1 / self.beta + np.sum(X @ self.w_var * X, axis=1)
|
||||
y_std = np.sqrt(y_var)
|
||||
return y, y_std
|
||||
return y
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
import numpy as np
|
||||
from prml.linear.logistic_regression import LogisticRegression
|
||||
|
||||
|
||||
class VariationalLogisticRegression(LogisticRegression):
|
||||
|
||||
def __init__(self, alpha:float=None, a0:float=1., b0:float=1.):
|
||||
"""
|
||||
construct variational logistic regressor
|
||||
|
||||
Parameters
|
||||
----------
|
||||
alpha : float
|
||||
precision parameter of the prior
|
||||
if None, this is also the subject to estimate
|
||||
a0 : float
|
||||
a parameter of hyper prior Gamma dist.
|
||||
Gamma(alpha|a0,b0)
|
||||
if alpha is not None, this argument will be ignored
|
||||
b0 : float
|
||||
another parameter of hyper prior Gamma dist.
|
||||
Gamma(alpha|a0,b0)
|
||||
if alpha is not None, this argument will be ignored
|
||||
"""
|
||||
if alpha is not None:
|
||||
self.__alpha = alpha
|
||||
else:
|
||||
self.a0 = a0
|
||||
self.b0 = b0
|
||||
|
||||
def fit(self, X:np.ndarray, t:np.ndarray, iter_max:int=1000):
|
||||
"""
|
||||
variational bayesian estimation of the parameter
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training independent variable
|
||||
t : (N,) np.ndarray
|
||||
training dependent variable
|
||||
iter_max : int, optional
|
||||
maximum number of iteration (the default is 1000)
|
||||
"""
|
||||
N, D = X.shape
|
||||
if hasattr(self, "a0"):
|
||||
self.a = self.a0 + 0.5 * D
|
||||
xi = np.random.uniform(-1, 1, size=N)
|
||||
I = np.eye(D)
|
||||
param = np.copy(xi)
|
||||
for _ in range(iter_max):
|
||||
lambda_ = np.tanh(xi) * 0.25 / xi
|
||||
self.w_var = np.linalg.inv(I / self.alpha + 2 * (lambda_ * X.T) @ X)
|
||||
self.w_mean = self.w_var @ np.sum(X.T * (t - 0.5), axis=1)
|
||||
xi = np.sqrt(np.sum(X @ (self.w_var + self.w_mean * self.w_mean[:, None]) * X, axis=-1))
|
||||
if np.allclose(xi, param):
|
||||
break
|
||||
else:
|
||||
param = np.copy(xi)
|
||||
|
||||
@property
|
||||
def alpha(self):
|
||||
if hasattr(self, "__alpha"):
|
||||
return self.__alpha
|
||||
else:
|
||||
try:
|
||||
self.b = self.b0 + 0.5 * (np.sum(self.w_mean ** 2) + np.trace(self.w_var))
|
||||
except AttributeError:
|
||||
self.b = self.b0
|
||||
return self.a / self.b
|
||||
|
||||
def proba(self, X:np.ndarray):
|
||||
"""
|
||||
compute probability of input belonging class 1
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : (N, D) np.ndarray
|
||||
training data independent variable
|
||||
|
||||
Returns
|
||||
-------
|
||||
(N,) np.ndarray
|
||||
probability of positive
|
||||
"""
|
||||
mu_a = X @ self.w_mean
|
||||
var_a = np.sum(X @ self.w_var * X, axis=1)
|
||||
y = self._sigmoid(mu_a / np.sqrt(1 + np.pi * var_a / 8))
|
||||
return y
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
from .categorical_hmm import CategoricalHMM
|
||||
from .gaussian_hmm import GaussianHMM
|
||||
from prml.markov.kalman import Kalman, kalman_filter, kalman_smoother
|
||||
from .particle import Particle
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GaussianHMM",
|
||||
"CategoricalHMM",
|
||||
"Kalman",
|
||||
"kalman_filter",
|
||||
"kalman_smoother",
|
||||
"Particle"
|
||||
]
|
||||
@@ -0,0 +1,65 @@
|
||||
import numpy as np
|
||||
from .hmm import HiddenMarkovModel
|
||||
|
||||
|
||||
class CategoricalHMM(HiddenMarkovModel):
|
||||
"""
|
||||
Hidden Markov Model with categorical emission model
|
||||
"""
|
||||
|
||||
def __init__(self, initial_proba, transition_proba, means):
|
||||
"""
|
||||
construct hidden markov model with categorical emission model
|
||||
|
||||
Parameters
|
||||
----------
|
||||
initial_proba : (n_hidden,) np.ndarray
|
||||
probability of initial latent state
|
||||
transition_proba : (n_hidden, n_hidden) np.ndarray
|
||||
transition probability matrix
|
||||
(i, j) component denotes the transition probability from i-th to j-th hidden state
|
||||
means : (n_hidden, ndim) np.ndarray
|
||||
mean parameters of categorical distribution
|
||||
|
||||
Returns
|
||||
-------
|
||||
ndim : int
|
||||
number of observation categories
|
||||
n_hidden : int
|
||||
number of hidden states
|
||||
"""
|
||||
assert initial_proba.size == transition_proba.shape[0] == transition_proba.shape[1] == means.shape[0]
|
||||
assert np.allclose(means.sum(axis=1), 1)
|
||||
super().__init__(initial_proba, transition_proba)
|
||||
self.ndim = means.shape[1]
|
||||
self.means = means
|
||||
|
||||
def draw(self, n=100):
|
||||
"""
|
||||
draw random sequence from this model
|
||||
|
||||
Parameters
|
||||
----------
|
||||
n : int
|
||||
length of the random sequence
|
||||
|
||||
Returns
|
||||
-------
|
||||
seq : (n,) np.ndarray
|
||||
generated random sequence
|
||||
"""
|
||||
hidden_state = np.random.choice(self.n_hidden, p=self.initial_proba)
|
||||
seq = []
|
||||
while len(seq) < n:
|
||||
seq.append(np.random.choice(self.ndim, p=self.means[hidden_state]))
|
||||
hidden_state = np.random.choice(self.n_hidden, p=self.transition_proba[hidden_state])
|
||||
return np.asarray(seq)
|
||||
|
||||
def likelihood(self, X):
|
||||
return self.means[X]
|
||||
|
||||
def maximize(self, seq, p_hidden, p_transition):
|
||||
self.initial_proba = p_hidden[0] / np.sum(p_hidden[0])
|
||||
self.transition_proba = np.sum(p_transition, axis=0) / np.sum(p_transition, axis=(0, 2))
|
||||
x = p_hidden[:, None, :] * (np.eye(self.ndim)[seq])[:, :, None]
|
||||
self.means = np.sum(x, axis=0) / np.sum(p_hidden, axis=0)
|
||||
@@ -0,0 +1,76 @@
|
||||
import numpy as np
|
||||
from prml.rv import MultivariateGaussian
|
||||
from .hmm import HiddenMarkovModel
|
||||
|
||||
|
||||
class GaussianHMM(HiddenMarkovModel):
|
||||
"""
|
||||
Hidden Markov Model with Gaussian emission model
|
||||
"""
|
||||
|
||||
def __init__(self, initial_proba, transition_proba, means, covs):
|
||||
"""
|
||||
construct hidden markov model with Gaussian emission model
|
||||
|
||||
Parameters
|
||||
----------
|
||||
initial_proba : (n_hidden,) np.ndarray or None
|
||||
probability of initial states
|
||||
transition_proba : (n_hidden, n_hidden) np.ndarray or None
|
||||
transition probability matrix
|
||||
(i, j) component denotes the transition probability from i-th to j-th hidden state
|
||||
means : (n_hidden, ndim) np.ndarray
|
||||
mean of each gaussian component
|
||||
covs : (n_hidden, ndim, ndim) np.ndarray
|
||||
covariance matrix of each gaussian component
|
||||
|
||||
Attributes
|
||||
----------
|
||||
ndim : int
|
||||
dimensionality of observation space
|
||||
n_hidden : int
|
||||
number of hidden states
|
||||
"""
|
||||
assert initial_proba.size == transition_proba.shape[0] == transition_proba.shape[1] == means.shape[0] == covs.shape[0]
|
||||
assert means.shape[1] == covs.shape[1] == covs.shape[2]
|
||||
super().__init__(initial_proba, transition_proba)
|
||||
self.ndim = means.shape[1]
|
||||
self.means = means
|
||||
self.covs = covs
|
||||
self.precisions = np.linalg.inv(self.covs)
|
||||
self.gaussians = [MultivariateGaussian(m, cov) for m, cov in zip(means, covs)]
|
||||
|
||||
def draw(self, n=100):
|
||||
"""
|
||||
draw random sequence from this model
|
||||
|
||||
Parameters
|
||||
----------
|
||||
n : int
|
||||
length of the random sequence
|
||||
|
||||
Returns
|
||||
-------
|
||||
seq : (n, ndim) np.ndarray
|
||||
generated random sequence
|
||||
"""
|
||||
hidden_state = np.random.choice(self.n_hidden, p=self.initial_proba)
|
||||
seq = []
|
||||
while len(seq) < n:
|
||||
seq.extend(self.gaussians[hidden_state].draw())
|
||||
hidden_state = np.random.choice(self.n_hidden, p=self.transition_proba[hidden_state])
|
||||
return np.asarray(seq)
|
||||
|
||||
def likelihood(self, X):
|
||||
diff = X[:, None, :] - self.means
|
||||
exponents = np.sum(
|
||||
np.einsum('nki,kij->nkj', diff, self.precisions) * diff, axis=-1)
|
||||
return np.exp(-0.5 * exponents) / np.sqrt(np.linalg.det(self.covs) * (2 * np.pi) ** self.ndim)
|
||||
|
||||
def maximize(self, seq, p_hidden, p_transition):
|
||||
self.initial_proba = p_hidden[0] / np.sum(p_hidden[0])
|
||||
self.transition_proba = np.sum(p_transition, axis=0) / np.sum(p_transition, axis=(0, 2))
|
||||
Nk = np.sum(p_hidden, axis=0)
|
||||
self.means = (seq.T @ p_hidden / Nk).T
|
||||
diffs = seq[:, None, :] - self.means
|
||||
self.covs = np.einsum('nki,nkj->kij', diffs, diffs * p_hidden[:, :, None]) / Nk[:, None, None]
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
class HiddenMarkovModel(object):
|
||||
"""
|
||||
Base class of Hidden Markov models
|
||||
"""
|
||||
|
||||
def __init__(self, initial_proba, transition_proba):
|
||||
"""
|
||||
construct hidden markov model
|
||||
|
||||
Parameters
|
||||
----------
|
||||
initial_proba : (n_hidden,) np.ndarray
|
||||
initial probability of each hidden state
|
||||
transition_proba : (n_hidden, n_hidden) np.ndarray
|
||||
transition probability matrix
|
||||
(i, j) component denotes the transition probability from i-th to j-th hidden state
|
||||
|
||||
Attribute
|
||||
---------
|
||||
n_hidden : int
|
||||
number of hidden state
|
||||
"""
|
||||
self.n_hidden = initial_proba.size
|
||||
self.initial_proba = initial_proba
|
||||
self.transition_proba = transition_proba
|
||||
|
||||
def fit(self, seq, iter_max=100):
|
||||
"""
|
||||
perform EM algorithm to estimate parameter of emission model and hidden variables
|
||||
|
||||
Parameters
|
||||
----------
|
||||
seq : (N, ndim) np.ndarray
|
||||
observed sequence
|
||||
iter_max : int
|
||||
maximum number of EM steps
|
||||
|
||||
Returns
|
||||
-------
|
||||
posterior : (N, n_hidden) np.ndarray
|
||||
posterior distribution of each latent variable
|
||||
"""
|
||||
params = np.hstack(
|
||||
(self.initial_proba.ravel(), self.transition_proba.ravel()))
|
||||
for i in range(iter_max):
|
||||
p_hidden, p_transition = self.expect(seq)
|
||||
self.maximize(seq, p_hidden, p_transition)
|
||||
params_new = np.hstack(
|
||||
(self.initial_proba.ravel(), self.transition_proba.ravel()))
|
||||
if np.allclose(params, params_new):
|
||||
break
|
||||
else:
|
||||
params = params_new
|
||||
return self.forward_backward(seq)
|
||||
|
||||
def expect(self, seq):
|
||||
"""
|
||||
estimate posterior distributions of hidden states and
|
||||
transition probability between adjacent latent variables
|
||||
|
||||
Parameters
|
||||
----------
|
||||
seq : (N, ndim) np.ndarray
|
||||
observed sequence
|
||||
|
||||
Returns
|
||||
-------
|
||||
p_hidden : (N, n_hidden) np.ndarray
|
||||
posterior distribution of each hidden variable
|
||||
p_transition : (N - 1, n_hidden, n_hidden) np.ndarray
|
||||
posterior transition probability between adjacent latent variables
|
||||
"""
|
||||
likelihood = self.likelihood(seq)
|
||||
|
||||
f = self.initial_proba * likelihood[0]
|
||||
constant = [f.sum()]
|
||||
forward = [f / f.sum()]
|
||||
for like in likelihood[1:]:
|
||||
f = forward[-1] @ self.transition_proba * like
|
||||
constant.append(f.sum())
|
||||
forward.append(f / f.sum())
|
||||
forward = np.asarray(forward)
|
||||
constant = np.asarray(constant)
|
||||
|
||||
backward = [np.ones(self.n_hidden)]
|
||||
for like, c in zip(likelihood[-1:0:-1], constant[-1:0:-1]):
|
||||
backward.insert(0, self.transition_proba @ (like * backward[0]) / c)
|
||||
backward = np.asarray(backward)
|
||||
|
||||
p_hidden = forward * backward
|
||||
p_transition = self.transition_proba * likelihood[1:, None, :] * backward[1:, None, :] * forward[:-1, :, None]
|
||||
return p_hidden, p_transition
|
||||
|
||||
def forward_backward(self, seq):
|
||||
"""
|
||||
estimate posterior distributions of hidden states
|
||||
|
||||
Parameters
|
||||
----------
|
||||
seq : (N, ndim) np.ndarray
|
||||
observed sequence
|
||||
|
||||
Returns
|
||||
-------
|
||||
posterior : (N, n_hidden) np.ndarray
|
||||
posterior distribution of hidden states
|
||||
"""
|
||||
likelihood = self.likelihood(seq)
|
||||
|
||||
f = self.initial_proba * likelihood[0]
|
||||
constant = [f.sum()]
|
||||
forward = [f / f.sum()]
|
||||
for like in likelihood[1:]:
|
||||
f = forward[-1] @ self.transition_proba * like
|
||||
constant.append(f.sum())
|
||||
forward.append(f / f.sum())
|
||||
|
||||
backward = [np.ones(self.n_hidden)]
|
||||
for like, c in zip(likelihood[-1:0:-1], constant[-1:0:-1]):
|
||||
backward.insert(0, self.transition_proba @ (like * backward[0]) / c)
|
||||
|
||||
forward = np.asarray(forward)
|
||||
backward = np.asarray(backward)
|
||||
posterior = forward * backward
|
||||
return posterior
|
||||
|
||||
def filtering(self, seq):
|
||||
"""
|
||||
bayesian filtering
|
||||
|
||||
Parameters
|
||||
----------
|
||||
seq : (N, ndim) np.ndarray
|
||||
observed sequence
|
||||
|
||||
Returns
|
||||
-------
|
||||
posterior : (N, n_hidden) np.ndarray
|
||||
posterior distributions of each latent variables
|
||||
"""
|
||||
likelihood = self.likelihood(seq)
|
||||
p = self.initial_proba * likelihood[0]
|
||||
posterior = [p / np.sum(p)]
|
||||
for like in likelihood[1:]:
|
||||
p = posterior[-1] @ self.transition_proba * like
|
||||
posterior.append(p / np.sum(p))
|
||||
posterior = np.asarray(posterior)
|
||||
return posterior
|
||||
|
||||
def viterbi(self, seq):
|
||||
"""
|
||||
viterbi algorithm (a.k.a. max-sum algorithm)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
seq : (N, ndim) np.ndarray
|
||||
observed sequence
|
||||
|
||||
Returns
|
||||
-------
|
||||
seq_hid : (N,) np.ndarray
|
||||
the most probable sequence of hidden variables
|
||||
"""
|
||||
nll = -np.log(self.likelihood(seq))
|
||||
cost_total = nll[0]
|
||||
from_list = []
|
||||
for i in range(1, len(seq)):
|
||||
cost_temp = cost_total[:, None] - np.log(self.transition_proba) + nll[i]
|
||||
cost_total = np.min(cost_temp, axis=0)
|
||||
index = np.argmin(cost_temp, axis=0)
|
||||
from_list.append(index)
|
||||
seq_hid = [np.argmin(cost_total)]
|
||||
for source in from_list[::-1]:
|
||||
seq_hid.insert(0, source[seq_hid[0]])
|
||||
return seq_hid
|
||||
+268
@@ -0,0 +1,268 @@
|
||||
import numpy as np
|
||||
from prml.rv.multivariate_gaussian import MultivariateGaussian as Gaussian
|
||||
from prml.markov.state_space_model import StateSpaceModel
|
||||
|
||||
|
||||
class Kalman(StateSpaceModel):
|
||||
"""
|
||||
A class to perform kalman filtering or smoothing
|
||||
z : internal state
|
||||
x : observation
|
||||
|
||||
z_1 ~ N(z_1|mu_0, P_0)\n
|
||||
z_n ~ N(z_n|A z_n-1, P)\n
|
||||
x_n ~ N(x_n|C z_n, S)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
system : (Dz, Dz) np.ndarray
|
||||
system matrix aka transition matrix (A)
|
||||
cov_system : (Dz, Dz) np.ndarray
|
||||
covariance matrix of process noise
|
||||
measure : (Dx, Dz) np.ndarray
|
||||
measurement matrix aka observation matrix (C)
|
||||
cov_measure : (Dx, Dx) np.ndarray
|
||||
covariance matrix of measurement noise
|
||||
mu0 : (Dz,) np.ndarray
|
||||
mean parameter of initial hidden variable
|
||||
P0 : (Dz, Dz) np.ndarray
|
||||
covariance parameter of initial hidden variable
|
||||
|
||||
Attributes
|
||||
----------
|
||||
Dz : int
|
||||
dimensionality of hidden variable
|
||||
Dx : int
|
||||
dimensionality of observed variable
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self, system, cov_system, measure, cov_measure, mu0, P0):
|
||||
"""
|
||||
construct Kalman model
|
||||
|
||||
z_1 ~ N(z_1|mu_0, P_0)\n
|
||||
z_n ~ N(z_n|A z_n-1, P)\n
|
||||
x_n ~ N(x_n|C z_n, S)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
system : (Dz, Dz) np.ndarray
|
||||
system matrix aka transition matrix (A)
|
||||
cov_system : (Dz, Dz) np.ndarray
|
||||
covariance matrix of process noise
|
||||
measure : (Dx, Dz) np.ndarray
|
||||
measurement matrix aka observation matrix (C)
|
||||
cov_measure : (Dx, Dx) np.ndarray
|
||||
covariance matrix of measurement noise
|
||||
mu0 : (Dz,) np.ndarray
|
||||
mean parameter of initial hidden variable
|
||||
P0 : (Dz, Dz) np.ndarray
|
||||
covariance parameter of initial hidden variable
|
||||
|
||||
Attributes
|
||||
----------
|
||||
hidden_mean : list of (Dz,) np.ndarray
|
||||
list of mean of hidden state starting from the given hidden state
|
||||
hidden_cov : list of (Dz, Dz) np.ndarray
|
||||
list of covariance of hidden state starting from the given hidden state
|
||||
"""
|
||||
self.system = system
|
||||
self.cov_system = cov_system
|
||||
self.measure = measure
|
||||
self.cov_measure = cov_measure
|
||||
|
||||
self.hidden_mean = [mu0]
|
||||
self.hidden_cov = [P0]
|
||||
self.hidden_cov_predicted = [None]
|
||||
|
||||
self.smoothed_until = -1
|
||||
self.smoothing_gain = [None]
|
||||
|
||||
def predict(self):
|
||||
"""
|
||||
predict hidden state at current step given estimate at previous step
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple ((Dz,) np.ndarray, (Dz, Dz) np.ndarray)
|
||||
tuple of mean and covariance of the estimate at current step
|
||||
"""
|
||||
mu_prev, cov_prev = self.hidden_mean[-1], self.hidden_cov[-1]
|
||||
mu = self.system @ mu_prev
|
||||
cov = self.system @ cov_prev @ self.system.T + self.cov_system
|
||||
self.hidden_mean.append(mu)
|
||||
self.hidden_cov.append(cov)
|
||||
self.hidden_cov_predicted.append(np.copy(cov))
|
||||
return mu, cov
|
||||
|
||||
def filter(self, observed):
|
||||
"""
|
||||
bayesian update of current estimate given current observation
|
||||
|
||||
Parameters
|
||||
----------
|
||||
observed : (Dx,) np.ndarray
|
||||
current observation
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple ((Dz,) np.ndarray, (Dz, Dz) np.ndarray)
|
||||
tuple of mean and covariance of the updated estimate
|
||||
"""
|
||||
mu, cov = self.hidden_mean[-1], self.hidden_cov[-1]
|
||||
innovation = observed - self.measure @ mu
|
||||
cov_innovation = self.cov_measure + self.measure @ cov @ self.measure.T
|
||||
kalman_gain = np.linalg.solve(cov_innovation, self.measure @ cov).T
|
||||
mu += kalman_gain @ innovation
|
||||
cov -= kalman_gain @ self.measure @ cov
|
||||
return mu, cov
|
||||
|
||||
def filtering(self, observed_sequence):
|
||||
"""
|
||||
perform kalman filtering given observed sequence
|
||||
|
||||
Parameters
|
||||
----------
|
||||
observed_sequence : (T, Dx) np.ndarray
|
||||
sequence of observations
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple ((T, Dz) np.ndarray, (T, Dz, Dz) np.ndarray)
|
||||
seuquence of mean and covariance of hidden variable at each time step
|
||||
"""
|
||||
for obs in observed_sequence:
|
||||
self.predict()
|
||||
self.filter(obs)
|
||||
mean_sequence = np.asarray(self.hidden_mean[1:])
|
||||
cov_sequence = np.asarray(self.hidden_cov[1:])
|
||||
return mean_sequence, cov_sequence
|
||||
|
||||
def smooth(self):
|
||||
"""
|
||||
bayesian update of current estimate with future observations
|
||||
"""
|
||||
mean_smoothed_next = self.hidden_mean[self.smoothed_until]
|
||||
cov_smoothed_next = self.hidden_cov[self.smoothed_until]
|
||||
cov_pred_next = self.hidden_cov_predicted[self.smoothed_until]
|
||||
|
||||
self.smoothed_until -= 1
|
||||
mean = self.hidden_mean[self.smoothed_until]
|
||||
cov = self.hidden_cov[self.smoothed_until]
|
||||
gain = np.linalg.solve(cov_pred_next, self.system @ cov).T
|
||||
mean += gain @ (mean_smoothed_next - self.system @ mean)
|
||||
cov += gain @ (cov_smoothed_next - cov_pred_next) @ gain.T
|
||||
self.smoothing_gain.insert(0, gain)
|
||||
|
||||
def smoothing(self, observed_sequence:np.ndarray=None):
|
||||
"""
|
||||
perform Kalman smoothing (given observed sequence)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
observed_sequence : (T, Dx) np.ndarray, optional
|
||||
sequence of observation
|
||||
run Kalman filter if given (the default is None)
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple ((T, Dz) np.ndarray, (T, Dz, Dz) np.ndarray)
|
||||
sequence of mean and covariance of hidden variable at each time step
|
||||
"""
|
||||
if observed_sequence is not None:
|
||||
self.filtering(observed_sequence)
|
||||
while self.smoothed_until != -len(self.hidden_mean):
|
||||
self.smooth()
|
||||
mean_sequence = np.asarray(self.hidden_mean[1:])
|
||||
cov_sequence = np.asarray(self.hidden_cov[1:])
|
||||
return mean_sequence, cov_sequence
|
||||
|
||||
def update_parameter(self, observation_sequence):
|
||||
"""
|
||||
maximization step of EM algorithm
|
||||
"""
|
||||
mu0 = self.hidden_mean[1]
|
||||
P0 = self.hidden_cov[1]
|
||||
|
||||
Ezn = np.asarray(self.hidden_mean)
|
||||
Eznzn = np.asarray(self.hidden_cov) + Ezn[..., None] * Ezn[:, None, :]
|
||||
Eznzn_1 = np.einsum("nij,nkj->nik", self.hidden_cov[2:], self.smoothing_gain[1:-1]) + Ezn[2:, :, None] * Ezn[1:-1, None, :]
|
||||
self.system = np.linalg.solve(np.sum(Eznzn[2:], axis=0), np.sum(Eznzn_1, axis=0).T).T
|
||||
self.cov_system = np.mean(
|
||||
Eznzn[2:]
|
||||
- np.einsum("ij,nkj->nik", self.system, Eznzn_1)
|
||||
- np.einsum("nij,kj->nik", Eznzn_1, self.system)
|
||||
+ np.einsum("ij,njk,lk->nil", self.system, Eznzn[1:-1], self.system),
|
||||
axis=0
|
||||
)
|
||||
self.measure = np.linalg.solve(
|
||||
np.sum(Eznzn[1:], axis=0),
|
||||
np.sum(np.einsum("ni,nj->nij", Ezn[1:], observation_sequence), axis=0)
|
||||
).T
|
||||
self.cov_measure = np.mean(
|
||||
np.einsum("ni,nj->nij", observation_sequence, observation_sequence)
|
||||
- np.einsum("ij,nj,nk->nik", self.measure, Ezn[1:], observation_sequence)
|
||||
- np.einsum("ni,nj,kj->nik", observation_sequence, Ezn[1:], self.measure)
|
||||
+ np.einsum("ij,njk,lk->nil", self.measure, Eznzn[1:], self.measure),
|
||||
axis=0
|
||||
)
|
||||
return self.system, self.cov_system, self.measure, self.cov_measure, mu0, P0
|
||||
|
||||
def fit(self, sequence, max_iter=10):
|
||||
for _ in range(max_iter):
|
||||
kalman_smoother(self, sequence)
|
||||
param = self.update_parameter(sequence)
|
||||
self.__init__(*param)
|
||||
return kalman_smoother(self, sequence)
|
||||
|
||||
|
||||
def kalman_filter(kalman:Kalman, observed_sequence:np.ndarray)->tuple:
|
||||
"""
|
||||
perform kalman filtering given Kalman model and observed sequence
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kalman : Kalman
|
||||
Kalman model
|
||||
observed_sequence : (T, Dx) np.ndarray
|
||||
sequence of observations
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple ((T, Dz) np.ndarray, (T, Dz, Dz) np.ndarray)
|
||||
seuquence of mean and covariance of hidden variable at each time step
|
||||
"""
|
||||
for obs in observed_sequence:
|
||||
kalman.predict()
|
||||
kalman.filter(obs)
|
||||
mean_sequence = np.asarray(kalman.hidden_mean[1:])
|
||||
cov_sequence = np.asarray(kalman.hidden_cov[1:])
|
||||
return mean_sequence, cov_sequence
|
||||
|
||||
|
||||
def kalman_smoother(kalman:Kalman, observed_sequence:np.ndarray=None):
|
||||
"""
|
||||
perform Kalman smoothing given Kalman model (and observed sequence)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kalman : Kalman
|
||||
Kalman model
|
||||
observed_sequence : (T, Dx) np.ndarray, optional
|
||||
sequence of observation
|
||||
run Kalman filter if given (the default is None)
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple ((T, Dz) np.ndarray, (T, Dz, Dz) np.ndarray)
|
||||
seuqnce of mean and covariance of hidden variable at each time step
|
||||
"""
|
||||
|
||||
if observed_sequence is not None:
|
||||
kalman_filter(kalman, observed_sequence)
|
||||
while kalman.smoothed_until != -len(kalman.hidden_mean):
|
||||
kalman.smooth()
|
||||
mean_sequence = np.asarray(kalman.hidden_mean[1:])
|
||||
cov_sequence = np.asarray(kalman.hidden_cov[1:])
|
||||
return mean_sequence, cov_sequence
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user