Spaces:
Runtime error
Runtime error
File size: 862 Bytes
4adaa35 b903ee4 4adaa35 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 |
from typing import List
from diskcache import Cache
from utils.data_proto import Page
from utils.summary_utils import summarize
cache = Cache(directory='/.cache/tmp/summary', size_limit=int(1e9)) # 1GB
def get_summarize_from_cache(pages: List[Page]):
pages_summaries = []
uncached_pages = []
for page in pages:
id = page['id']
summary = cache.get(id)
if summary is not None:
pages_summaries.append(summary)
else:
uncached_pages.append(page)
return pages_summaries, uncached_pages
def summarize_un_cache_page(pages: List[Page]):
print(f'summarize pages length: {len(pages)}')
for page in pages:
id = page['id']
print(f'processing page: {id}')
summary = summarize(id, page['text'])
cache.set(id, summary)
print(f'processed page: {id}')
|