[shopify] skip consecutive duplicate products

Not filtering duplicate URLs anymore caused the archive ID uniqueness
test to fail.
pull/359/head
Mike Fährmann 5 years ago
parent b89f0d8d3c
commit bc6b0cfddc
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -89,10 +89,14 @@ class ShopifyCollectionExtractor(ShopifyExtractor):
while True:
page = self.request(self.item_url, params=params).text
urls = search_re.findall(page)
last = None
if not urls:
return
for path in urls:
if last == path:
continue
last = path
yield self.root + path
params["page"] += 1

Loading…
Cancel
Save