File size: 3,543 Bytes
e78f32a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 |
#Class to fetch news and stock data from the web for a specific ticker and combine them into a dataframe.
import pandas as pd
import yfinance as yf
from pygooglenews import GoogleNews
from transformers import AutoTokenizer, AutoModelForSequenceClassification
from transformers import pipeline
class InferenceDataPipeline:
def __init__(self, ticker, time_period_news, time_period_stock):
self.ticker = ticker
self.time_period_news = time_period_news
self.time_period_stock = time_period_stock
def get_data(self):
stock_data = self.get_stock_data()
news_data = self.get_news_data()
news_sentiment = self.get_sentiment(news_data)
combined_data = self.combine_data(stock_data, news_sentiment)
return combined_data
def get_stock_data(self):
data = yf.download(self.ticker , period = self.time_period_stock)
df = pd.DataFrame()
df['Open'] = data['Open']
df['Close'] = data['Close']
df['High'] = data['High']
df['Low'] = data['Low']
df['Volume'] = data['Volume']
return df
def get_news_data(self):
googlenews = GoogleNews()
news_data = googlenews.search(self.ticker, when=self.time_period_news)
news_data = pd.DataFrame(news_data['entries'])
return news_data
def get_sentiment(self, news_data):
tokenizer = AutoTokenizer.from_pretrained("ProsusAI/finbert")
model = AutoModelForSequenceClassification.from_pretrained("ProsusAI/finbert")
classifier = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
news_sentiment = []
for i in range(len(news_data)):
sentiment = classifier(news_data['title'][i], top_k=None)
postive_score = sentiment[0]['score']
negative_score = sentiment[1]['score']
neutral_score = sentiment[2]['score']
reformmated_time_stamp = pd.to_datetime(news_data['published'][i]).date()
news_sentiment.append({'Date': reformmated_time_stamp, 'positive_score': postive_score, 'negative_score': negative_score, 'neutral_score': neutral_score})
return pd.DataFrame(news_sentiment)
def combine_data(self, stock_data, news_sentiment):
news_sentiment = (
news_sentiment
.groupby('Date')
.mean()
.fillna(0)
.reset_index()
.set_index('Date')
.sort_index()
)
common_index = pd.date_range(
start=pd.Timestamp(min(pd.Timestamp(stock_data.index[0]), pd.Timestamp(news_sentiment.index[0]))),
end=pd.Timestamp(max(pd.Timestamp(stock_data.index[-1]), pd.Timestamp(news_sentiment.index[-1]))),
freq='D'
)
stock_data = stock_data.reindex(common_index).fillna(-1)
news_sentiment = news_sentiment.reindex(common_index).fillna(0)
#Ensure stock_data and news_sentiment have combatile indices
stock_data.index = pd.to_datetime(stock_data.index).normalize()
news_sentiment.index = pd.to_datetime(news_sentiment.index).normalize()
combined_data = pd.merge(
stock_data,
news_sentiment,
how='left',
left_index=True,
right_index=True
)
#Drop all close values that are -1
combined_data = combined_data[combined_data['Close'] != -1]
#fill all missing values with 0
combined_data = combined_data.fillna(0)
return combined_data |