|
|
|
@ -1,6 +1,6 @@
|
|
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
|
|
|
|
|
# Copyright 2014, 2015 Mike Fährmann
|
|
|
|
|
# Copyright 2014-2017 Mike Fährmann
|
|
|
|
|
#
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
|
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
|
|
@ -10,11 +10,15 @@
|
|
|
|
|
|
|
|
|
|
import time
|
|
|
|
|
import queue
|
|
|
|
|
import logging
|
|
|
|
|
import requests
|
|
|
|
|
import threading
|
|
|
|
|
from .message import Message
|
|
|
|
|
from .. import config
|
|
|
|
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Extractor():
|
|
|
|
|
|
|
|
|
|
category = ""
|
|
|
|
@ -92,4 +96,7 @@ def safe_request(session, url, method="GET", *args, **kwargs):
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
# everything ok -- proceed to download
|
|
|
|
|
log.debug(url)
|
|
|
|
|
log.debug(r.headers)
|
|
|
|
|
log.debug(r.text)
|
|
|
|
|
return r
|
|
|
|
|