Kevin Hu
commited on
Commit
·
bc6fc1a
1
Parent(s):
d007a79
smooth term weight (#3510)
Browse files### What problem does this PR solve?
#3499
### Type of change
- [x] Performance Improvement
- rag/nlp/term_weight.py +2 -2
rag/nlp/term_weight.py
CHANGED
@@ -228,7 +228,7 @@ class Dealer:
|
|
228 |
idf2 = np.array([idf(df(t), 1000000000) for t in tks])
|
229 |
wts = (0.3 * idf1 + 0.7 * idf2) * \
|
230 |
np.array([ner(t) * postag(t) for t in tks])
|
231 |
-
wts = [math.
|
232 |
tw = list(zip(tks, wts))
|
233 |
else:
|
234 |
for tk in tks:
|
@@ -237,7 +237,7 @@ class Dealer:
|
|
237 |
idf2 = np.array([idf(df(t), 1000000000) for t in tt])
|
238 |
wts = (0.3 * idf1 + 0.7 * idf2) * \
|
239 |
np.array([ner(t) * postag(t) for t in tt])
|
240 |
-
wts = [math.
|
241 |
tw.extend(zip(tt, wts))
|
242 |
|
243 |
S = np.sum([s for _, s in tw])
|
|
|
228 |
idf2 = np.array([idf(df(t), 1000000000) for t in tks])
|
229 |
wts = (0.3 * idf1 + 0.7 * idf2) * \
|
230 |
np.array([ner(t) * postag(t) for t in tks])
|
231 |
+
wts = [math.pow(s, 2) for s in wts]
|
232 |
tw = list(zip(tks, wts))
|
233 |
else:
|
234 |
for tk in tks:
|
|
|
237 |
idf2 = np.array([idf(df(t), 1000000000) for t in tt])
|
238 |
wts = (0.3 * idf1 + 0.7 * idf2) * \
|
239 |
np.array([ner(t) * postag(t) for t in tt])
|
240 |
+
wts = [math.pow(s, 2) for s in wts]
|
241 |
tw.extend(zip(tt, wts))
|
242 |
|
243 |
S = np.sum([s for _, s in tw])
|