<reponame>CodeBoy2006/AIWordcloud
# -*- coding: utf-8 -*-

import requests
from bs4 import BeautifulSoup
from newspaper import Article
import sys
import jieba.analyse
from os import path
import collections
import re

#用newspaper库智能爬取网页数据
def smartspider(url):
	try:
		news = Article(url, language='zh')

		news .download()  #先下载
		news .parse()    #再解析

		print('context= '+str(news.text)) #新闻正文
		return str(news.text)
	except:
		print('URL出错')
		return ''
	
#用 BeautifulSoup4 来爬取数据
def bs4spider(url):
	try:
		newsurl = url
		res = requests.get(newsurl)
		res.encoding = 'utf-8'

		soup = BeautifulSoup(res.text, 'html.parser')
		article = []
		for p in soup.select('.art_p')[:-1]:
			article.append(p.text.strip())
		''.join(article)
		print('context= '+str(article))
		return str(article)
	except:
		print('URL出错')
		return ''

#2019/12/18新增函数,经过本人剖析源代码,发现优化方案:
#该函数生成一个字典用于直接生成词云图,不需通过wordcloud库自带的字频统计功能,实现单字词云图，提高效率。
def make_fake_frequencies(textlist):
    frequencies = {}
    for n in textlist:
    	frequencies[n]=1 #由于事先经过统计,词频全部记为一对最终结果无影响

    return frequencies

def collection_characters(text,num):
	rule = re.compile(r"[^a-zA-Z0-9\u4e00-\u9fa5]")
	text = rule.sub('',text)
	mylist = list(str(text))
	mycount = collections.Counter(mylist)
	words = []
	reslut = {} #最终返回的数组由两部分组成,用于生成词云图的字典与便于用户识别的字符串
	for key, val in mycount.most_common(num):  # 有序（返回前150个）
		print(key, val)
		words.append(key)
	# print(reslut)
	# print(type(' '.join(reslut)))
	reslut['string']=' '.join(words)
	reslut['frequencies']=make_fake_frequencies(words)
	return reslut

def jiebaclearText(text,num):
	reslut = {} #最终返回的数组由两部分组成,用于生成词云图的字典与便于用户识别的字符串
	stopwords_path = 'data/stopwords/stopwords_baidu.txt' # 停用词词表
	#摒弃老旧的方法，使用 jieba 库智能提取前150个关键词
	jieba.analyse.set_stop_words(stopwords_path)
	tags = jieba.analyse.extract_tags(text,num)
	print(tags)
	reslut['string']=' '.join(tags)
	reslut['frequencies']=make_fake_frequencies(tags)
	return reslut

def split(string,mode,num):
	if mode == '字词词云图':
		reslut = jiebaclearText(string,num)
	else:
		reslut = collection_characters(string,num)

	print(reslut['string'])
	return reslut





# -*- coding: utf-8 -*-

import requests
from bs4 import BeautifulSoup
from newspaper import Article
import sys
import jieba.analyse
from os import path
import collections
import re

#用newspaper库智能爬取网页数据
def smartspider(url):
	try:
		news = Article(url, language='zh')

		news .download()  #先下载
		news .parse()    #再解析

		print('context= '+str(news.text)) #新闻正文
		return str(news.text)
	except:
		print('URL出错')
		return ''
	
#用 BeautifulSoup4 来爬取数据
def bs4spider(url):
	try:
		newsurl = url
		res = requests.get(newsurl)
		res.encoding = 'utf-8'

		soup = BeautifulSoup(res.text, 'html.parser')
		article = []
		for p in soup.select('.art_p')[:-1]:
			article.append(p.text.strip())
		''.join(article)
		print('context= '+str(article))
		return str(article)
	except:
		print('URL出错')
		return ''

#2019/12/18新增函数,经过本人剖析源代码,发现优化方案:
#该函数生成一个字典用于直接生成词云图,不需通过wordcloud库自带的字频统计功能,实现单字词云图，提高效率。
def make_fake_frequencies(textlist):
    frequencies = {}
    for n in textlist:
    	frequencies[n]=1 #由于事先经过统计,词频全部记为一对最终结果无影响

    return frequencies

def collection_characters(text,num):
	rule = re.compile(r"[^a-zA-Z0-9\u4e00-\u9fa5]")
	text = rule.sub('',text)
	mylist = list(str(text))
	mycount = collections.Counter(mylist)
	words = []
	reslut = {} #最终返回的数组由两部分组成,用于生成词云图的字典与便于用户识别的字符串
	for key, val in mycount.most_common(num):  # 有序（返回前150个）
		print(key, val)
		words.append(key)
	# print(reslut)
	# print(type(' '.join(reslut)))
	reslut['string']=' '.join(words)
	reslut['frequencies']=make_fake_frequencies(words)
	return reslut

def jiebaclearText(text,num):
	reslut = {} #最终返回的数组由两部分组成,用于生成词云图的字典与便于用户识别的字符串
	stopwords_path = 'data/stopwords/stopwords_baidu.txt' # 停用词词表
	#摒弃老旧的方法，使用 jieba 库智能提取前150个关键词
	jieba.analyse.set_stop_words(stopwords_path)
	tags = jieba.analyse.extract_tags(text,num)
	print(tags)
	reslut['string']=' '.join(tags)
	reslut['frequencies']=make_fake_frequencies(tags)
	return reslut

def split(string,mode,num):
	if mode == '字词词云图':
		reslut = jiebaclearText(string,num)
	else:
		reslut = collection_characters(string,num)

	print(reslut['string'])
	return reslut
# -*- coding: utf-8 -*- #用newspaper库智能爬取网页数据 #先下载 #再解析 #新闻正文 #用 BeautifulSoup4 来爬取数据 #2019/12/18新增函数,经过本人剖析源代码,发现优化方案: #该函数生成一个字典用于直接生成词云图,不需通过wordcloud库自带的字频统计功能,实现单字词云图，提高效率。 #由于事先经过统计,词频全部记为一对最终结果无影响 #最终返回的数组由两部分组成,用于生成词云图的字典与便于用户识别的字符串 # 有序（返回前150个） # print(reslut) # print(type(' '.join(reslut))) #最终返回的数组由两部分组成,用于生成词云图的字典与便于用户识别的字符串 # 停用词词表 #摒弃老旧的方法，使用 jieba 库智能提取前150个关键词