import scrapy from scrapy.crawler import CrawlerRunner from scrapy.settings import Settings from twisted.internet import reactor class ExampleSpider(scrapy.Spider): name = 'example' allowed_domains = ['example.com'] start_urls = [''] def parse(self, response): print(response.text) settings = Settings({'USER_AGENT': 'PYTHON'}) runner = CrawlerRunner(settings) d = runner.crawl(ExampleSpider) d.addBoth(lambda _: reactor.stop()) reactor.run()