win10子系统python开发环境准备及kenlm和nltk的使用教程

wget http://ftp.gnu.org/gnu/make/make-4.2.tar.gz
tar -zxvf make-4.2.tar.gz
cd make-4.2
./configure #生成 Makefile，为下一步的编译做准备
make     #进行源代码编译
sudo make install #安装,这一步会将编译好的make文件转移到/usr/local/make目录下，该目录需要root权限
ln -s -f /usr/local/bin/make /usr/bin/make
make --version #查看版本

wget https://cmake.org/files/v3.3/cmake-3.5.1.tar.gz
tar xzvf cmake-3.5.1.tar.gz
yum install gcc-c++ # 安装gcc等程序包(安装过就忽略)
cd cmake-3.5.1
./bootstrap
make
make install
cmake --version #查看版本

wget https://www.python.org/ftp/python/3.7.3/Python-3.7.3.tgz

tar zxvf Python-3.7.3.tgz
cd Python-3.7.3

./configure --with-ssl
make
sudo make install

# 添加python3的软链接
sudo ln -s /usr/local/bin/python3.7 /usr/bin/python3
# 添加 pip3 的软链接
sudo ln -s /usr/local/bin/pip3.7 /usr/bin/pip3

cd boost_1_70_0
./bootstrap.sh
./b2 install

wget http://tukaani.org/xz/xz-5.2.2.tar.gz
tar xzvf xz-5.2.2.tar.gz
cd xz-5.2.2
./configure
make
make install

wget http://zlib.net/zlib-1.2.11.tar.gz
tar xzf zlib-1.2.11.tar.gz
cd zlib-1.2.11
./configure
make
make install

wget https://fossies.org/linux/misc/bzip2-1.0.6.tar.gz
tar xzvf bzip2-1.0.6.tar.gz
cd bzip2-1.0.6/
make
make install

cd kenlm
mkdir -p build
cd build
cmake ..
make -j 4 # 启用4个cpu去编译。提高编译速度
cd ..
python setup.py install

import nltk
nltk.data.find(".")

# Searched in:
#   - '/root/nltk_data'
#   - '/usr/local/nltk_data'
#   - '/usr/local/share/nltk_data'
#   - '/usr/local/lib/nltk_data'
#   - '/usr/share/nltk_data'
#   - '/usr/local/share/nltk_data'
#   - '/usr/lib/nltk_data'
#   - '/usr/local/lib/nltk_data'

from nltk import data
data.path.append(r"你下载的nltk_data所在路径")

from nltk.tokenize import word_tokenize
sentence = "since the 1890s , and beginning in france , the term ''libertarianism '' has often been used as an synonym for anarchism and was used almost exclusively in this sense until the 1950s in the united states ; its use as an synonym is still common outside the united states ."
print(word_tokenize(sentence))