How to run spiders sequentially (one after another) with a 2-minute interval in my task?
signals.py
@receiver(post_save, sender=ParseCategoryUrl)
def start_parse_from_category_url(sender, created, instance, **kwargs):
"""
The start_parse_from_category_url function is a signal receiver that
runs the run_spider_to_pars_ads_list task
when an instance of CategoryUrl model is created.
"""
if instance.url:
group(
run_spider_to_pars_ads_list.s(url=instance.url, user=instance.user.id),
).apply_async()
tasks.py
@app.task(name="run_spider_to_pars_ads_list")
def run_spider_to_pars_ads_list(
url: str, user: int, pages: Optional[int] = None
) -> None:
process.crawl(ListitemsSpider, url=url, user=user, pages=pages)
d = process.join()
d.addBoth(lambda _: reactor.stop())
reactor.run()
