add corpus, dict and mdel

app.py

@@ -0,0 +1,28 @@
+import numpy as np
+import pandas as pd
+from gensim.corpora import Dictionary, MmCorpus
+from gensim.models import LdaModel, Word2Vec
+import matplotlib.pyplot as plt
+import streamlit as st
+from pyLDAvis import prepared_data_to_html
+import pyLDAvis.gensim_models as gensimvis
+
+# 生データ・コーパス・辞書・モデルのロード
+df = pd.read_csv("./raw_corpus.csv")
+corpus = MmCorpus('./corpus.mm')
+dict = Dictionary.load(f'./livedoor_demo.dict')
+lda = LdaModel.load('./lda_demo.model')
+
+st.caption("生データ一覧")
+st.dataframe(df.iloc[:100])
+
+st.caption("記事のカテゴリ")
+fig, ax = plt.subplots()
+count = df[["CATEGORY", "DOCUMENT"]].groupby("CATEGORY").count()
+count.plot.pie(y="DOCUMENT", ax=ax, ylabel="", legend=False)
+st.pyplot(fig)
+
+# pyLDAvisによるトピックの可視化
+vis = gensimvis.prepare(lda, corpus, dict)
+html_string = prepared_data_to_html(vis)
+st.components.v1.html(html_string, width=1300, height=800)

corpus.mm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38635f82ae9f3c4dda7102fc9490bdedc7252db2ce112567fa770ca12026e073
+size 11175678

corpus.mm.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72767db46c1cd1d6fa978d07d3292ab8f05f568ea07b2fb6e2bf5360a68a2f1c
+size 36780

lda_demo.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:343b994ba88549e98bf9005b12aaa905a7fbd8799da33c56eaf239bbef306767
+size 161307

lda_demo.model.expElogbeta.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67b82332c681036407bff78f2e60e8317ea114c96037edd52e2296222e2e650a
+size 1416296

lda_demo.model.id2word

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2e99dcee6c62d468b5d64053a2654a3559ea6cad38c5ca934ee4db5e91d14f6
+size 1283235

lda_demo.model.state

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4d1a044ffad64e9769daf521cf6195c74a6f221332f3c1435ac0aca167f136d
+size 1574192

livedoor_demo.dict

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d7019ad7b4fd3f486a1ea96f65ce5a2c2c0fd8e0d56cb38a4f9beac1850fb14
+size 1283368

raw_corpus.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:746107980f471c893fa8fb1fb40e65f50bbf9b97fe5f7ad485e8d6997b9baf7f
+size 26062717

requirements.txt

@@ -0,0 +1,69 @@
+altair==4.2.2
+attrs==22.2.0
+blinker==1.5
+cachetools==5.3.0
+certifi==2022.12.7
+charset-normalizer==3.0.1
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+decorator==5.1.1
+entrypoints==0.4
+fonttools==4.38.0
+fst-pso==1.8.1
+funcy==1.18
+future==0.18.3
+FuzzyTM==2.0.5
+gensim==4.3.0
+gitdb==4.0.10
+GitPython==3.1.31
+idna==3.4
+importlib-metadata==6.0.0
+importlib-resources==5.12.0
+Jinja2==3.1.2
+joblib==1.2.0
+jsonschema==4.17.3
+kiwisolver==1.4.4
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.0
+mdurl==0.1.2
+miniful==0.0.6
+numexpr==2.8.4
+numpy==1.24.2
+packaging==23.0
+pandas==1.5.3
+Pillow==9.4.0
+protobuf==3.20.3
+pyarrow==11.0.0
+pydeck==0.8.0
+pyFUME==0.2.25
+Pygments==2.14.0
+pyLDAvis==3.3.1
+Pympler==1.0.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+pytz==2022.7.1
+pytz-deprecation-shim==0.1.0.post0
+requests==2.28.2
+rich==13.3.1
+scikit-learn==1.2.1
+scipy==1.10.0
+semver==2.13.0
+simpful==2.9.0
+six==1.16.0
+sklearn==0.0.post1
+smart-open==6.3.0
+smmap==5.0.0
+streamlit==1.19.0
+threadpoolctl==3.1.0
+toml==0.10.2
+toolz==0.12.0
+tornado==6.2
+typing_extensions==4.5.0
+tzdata==2022.7
+tzlocal==4.2
+urllib3==1.26.14
+validators==0.20.0
+zipp==3.14.0

word2vec.gensim.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9125139e4c1394e42e1c792f2ff5703a6180e9ac38b6c0122cc9397115eccd48
+size 23527420

word2vec.gensim.model.syn1neg.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:164b5cd74b4abc03e7090eae31ceae68e8758aea374b07af7415bc789dcaf4be
+size 67095280

word2vec.gensim.model.wv.syn0.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cca0fc3f50b79eb3518ce84ae54019767e41815d8d31748a00f77be650edbb1
+size 67095280

word2vec.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fa90bdfdcca128e8d6689ff883ab8d3e587ab6b348f7bed7f1ad13a50782b12
+size 21550345