scrapy.core.downloader.Downloader

class documentation

class Downloader: (source)

View In Hierarchy

Undocumented

Method __init__ Undocumented
Method close Undocumented
Method fetch Undocumented
Method needs_backout Undocumented
Constant DOWNLOAD_SLOT Undocumented
Instance Variable active Undocumented
Instance Variable domain_concurrency Undocumented
Instance Variable handlers Undocumented
Instance Variable ip_concurrency Undocumented
Instance Variable middleware Undocumented
Instance Variable randomize_delay Undocumented
Instance Variable settings Undocumented
Instance Variable signals Undocumented
Instance Variable slots Undocumented
Instance Variable total_concurrency Undocumented
Method _download Undocumented
Method _enqueue_request Undocumented
Method _get_slot Undocumented
Method _get_slot_key Undocumented
Method _process_queue Undocumented
Method _slot_gc Undocumented
Instance Variable _slot_gc_loop Undocumented
def __init__(self, crawler): (source)

Undocumented

def close(self): (source)

Undocumented

def fetch(self, request, spider): (source)

Undocumented

def needs_backout(self): (source)

Undocumented

DOWNLOAD_SLOT: str = (source)

Undocumented

Value
'download_slot'
active = (source)

Undocumented

domain_concurrency = (source)

Undocumented

handlers = (source)

Undocumented

ip_concurrency = (source)

Undocumented

middleware = (source)

Undocumented

randomize_delay = (source)

Undocumented

settings = (source)

Undocumented

signals = (source)

Undocumented

slots: dict = (source)

Undocumented

total_concurrency = (source)

Undocumented

def _download(self, slot, request, spider): (source)

Undocumented

def _enqueue_request(self, request, spider): (source)

Undocumented

def _get_slot(self, request, spider): (source)

Undocumented

def _get_slot_key(self, request, spider): (source)

Undocumented

def _process_queue(self, spider, slot): (source)

Undocumented

def _slot_gc(self, age=60): (source)

Undocumented

_slot_gc_loop = (source)

Undocumented