[nozomi] reduce memory consumption during searches (#2754)

only load and use the entire 'index.nozomi' database
if there are only negative search terms
pull/2757/head
Mike Fährmann 2 years ago
parent 467a2a4d35
commit 62cc47755b
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -193,25 +193,27 @@ class NozomiSearchExtractor(NozomiExtractor):
return {"search_tags": self.tags} return {"search_tags": self.tags}
def posts(self): def posts(self):
index = None result = None
result = set()
def nozomi(path): def nozomi(path):
url = "https://j.nozomi.la/" + path + ".nozomi" url = "https://j.nozomi.la/" + path + ".nozomi"
return decode_nozomi(self.request(url).content) return decode_nozomi(self.request(url).content)
positive, negative = [], []
for tag in self.tags: for tag in self.tags:
tag = tag.replace("/", "") (negative if tag[0] == "-" else positive).append(
if tag[0] == "-": tag.replace("/", ""))
if not index:
index = set(nozomi("index"))
items = index.difference(nozomi("nozomi/" + tag[1:]))
else:
items = nozomi("nozomi/" + tag)
if result: for tag in positive:
result.intersection_update(items) ids = nozomi("nozomi/" + tag)
if result is None:
result = set(ids)
else: else:
result.update(items) result.intersection_update(ids)
for tag in negative:
if result is None:
result = set(nozomi("index"))
result.difference_update(nozomi("nozomi/" + tag[1:]))
return sorted(result, reverse=True) return sorted(result, reverse=True) if result else ()

Loading…
Cancel
Save