update some extractors to use https

8 years ago · 13dc5d72bc
parent 342371086b
commit 13dc5d72bc
12 changed files with 43 additions and 41 deletions
--- a/gallery_dl/extractor/deviantart.py
+++ b/gallery_dl/extractor/deviantart.py
@ -6,7 +6,7 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.

-"""Extract images from http://www.deviantart.com/"""
+"""Extract images from https://www.deviantart.com/"""

 from .common import Extractor, Message
 from .. import text, exception
@ -107,9 +107,9 @@ class DeviantartImageExtractor(DeviantartExtractor):
        return (self.api.deviation(deviation_id),)


-class DeviantartFavouritesExtractor(DeviantartExtractor):
+class DeviantartFavoriteExtractor(DeviantartExtractor):
    """Extractor for an artist's favourites from deviantart.com"""
-    subcategory = "favourites"
+    subcategory = "favorite"
    directory_fmt = ["{category}", "{subcategory}",
                     "{collection[owner]} - {collection[title]}"]
    pattern = [r"(?:https?://)?([^\.]+)\.deviantart\.com/favourites"
@ -117,12 +117,12 @@ class DeviantartFavouritesExtractor(DeviantartExtractor):
    test = [
        ("http://h3813067.deviantart.com/favourites/", {
            "url": "71345ce3bef5b19bd2a56d7b96e6b5ddba747c2e",
-            "keyword": "35a275b0f737aa9bd1f32ba13604d6e9a7054a14",
+            "keyword": "51e88d400c3fb69ae0b5a618ef21a282697185fe",
            "content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e",
        }),
        ("http://rosuuri.deviantart.com/favourites/58951174/Useful", {
            "url": "9e8d971c80db099b95d1c785399e2bc6eb96cd07",
-            "keyword": "cf65309a880799a4a82a7b2f0389e5bc88f5730f",
+            "keyword": "ea42a4e238f7e54caa250fa2f5abef103cbf30ab",
        }),
    ]

--- a/gallery_dl/extractor/dynastyscans.py
+++ b/gallery_dl/extractor/dynastyscans.py
@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-

-# Copyright 2015 Mike Fährmann
+# Copyright 2015-2017 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.

-"""Extract manga-chapters from http://dynasty-scans.com/"""
+"""Extract manga-chapters from https://dynasty-scans.com/"""

 from .common import Extractor, Message
 from .. import text
@ -24,16 +24,16 @@ class DynastyscansChapterExtractor(Extractor):
    test = [
        (("http://dynasty-scans.com/chapters/"
          "hitoribocchi_no_oo_seikatsu_ch33"), {
-            "url": "63950fa1dfdef58ab842c1b9b854c5c1d650cfa0",
+            "url": "ba945116382eff26b0e52fe1804e6d58fee7734d",
            "keyword": "81bfda5b98b34ac2a7324bd9e2abad3df9cc7673",
        }),
        (("http://dynasty-scans.com/chapters/"
          "new_game_the_spinoff_special_13"), {
-            "url": "6b28c733481ac498da341e85a9eb155864491731",
+            "url": "2cd5e04bd16f842dc884c145a44cf0c64ec27a21",
            "keyword": "93b75d0c0aaeb849c99f2225a4b97f466bc3ace9",
        }),
    ]
-    url_base = "http://dynasty-scans.com/"
+    url_base = "https://dynasty-scans.com/"

    def __init__(self, match):
        Extractor.__init__(self)
--- a/gallery_dl/extractor/gelbooru.py
+++ b/gallery_dl/extractor/gelbooru.py
@ -6,7 +6,7 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.

-"""Extract images from http://gelbooru.com/"""
+"""Extract images from https://gelbooru.com/"""

 from . import booru
 from .. import config
@ -15,7 +15,7 @@ from .. import config
 class GelbooruExtractor(booru.XMLBooruExtractor):
    """Base class for gelbooru extractors"""
    category = "gelbooru"
-    api_url = "http://gelbooru.com/"
+    api_url = "https://gelbooru.com/"
    pagestart = 0
    pagekey = "pid"

--- a/gallery_dl/extractor/hentaifoundry.py
+++ b/gallery_dl/extractor/hentaifoundry.py
@ -6,7 +6,7 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.

-"""Extract images from http://www.hentai-foundry.com/"""
+"""Extract images from https://www.hentai-foundry.com/"""

 from .common import Extractor, Message
 from .. import text, exception
@ -25,15 +25,15 @@ class HentaifoundryUserExtractor(Extractor):
         r"user/([^/]+)/profile"),
    ]
    test = [
-        ("http://www.hentai-foundry.com/pictures/user/Tenpura", {
-            "url": "35124cf236ffec596092446322b8f0ad603571c5",
+        ("https://www.hentai-foundry.com/pictures/user/Tenpura", {
+            "url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28",
            "keyword": "6e9a549feb9bafebd9d9342ef3c8ccad33a7031c",
        }),
        ("http://www.hentai-foundry.com/user/asdq/profile", {
            "exception": exception.NotFoundError,
        }),
    ]
-    url_base = "http://www.hentai-foundry.com/pictures/user/"
+    url_base = "https://www.hentai-foundry.com/pictures/user/"

    def __init__(self, match):
        Extractor.__init__(self)
@ -85,7 +85,7 @@ class HentaifoundryUserExtractor(Extractor):
            page, '//pictures.hentai-foundry.com', '"', pos)
        data = {"index": index, "title": text.unescape(title)}
        text.nameext_from_url(url, data)
-        return "http://pictures.hentai-foundry.com" + url, data
+        return "https://pictures.hentai-foundry.com" + url, data

    def set_filters(self, token):
        """Set site-internal filters to show all images"""
@ -107,11 +107,14 @@ class HentaifoundryUserExtractor(Extractor):
            "rating_female": 1,
            "rating_futa": 1,
            "rating_other": 1,
+            "rating_scat": 1,
+            "rating_incest": 1,
+            "rating_rape": 1,
            "filter_media": "A",
            "filter_order": "date_new",
            "filter_type": 0,
        }
-        self.request("http://www.hentai-foundry.com/site/filters",
+        self.request("https://www.hentai-foundry.com/site/filters",
                     method="post", data=formdata)


@ -127,7 +130,7 @@ class HentaifoundryImageExtractor(Extractor):
    test = [
        (("http://www.hentai-foundry.com/"
          "pictures/user/Tenpura/407501/shimakaze"), {
-            "url": "b68d1b0121b97e01a878beeb2e43b07cb881b5a9",
+            "url": "fbf2fd74906738094e2575d2728e8dc3de18a8a3",
            "keyword": "304479cfe00fbb723886be78b2bd6b9306a31d8a",
            "content": "91bf01497c39254b6dfb234a18e8f01629c77fd1",
        }),
@ -149,7 +152,7 @@ class HentaifoundryImageExtractor(Extractor):

    def get_image_metadata(self):
        """Collect metadata for an image"""
-        url = "http://www.hentai-foundry.com/pictures/user/{}/{}".format(
+        url = "https://www.hentai-foundry.com/pictures/user/{}/{}".format(
            self.artist, self.index)
        response = self.session.get(url + "?enterAgree=1")
        if response.status_code == 404:
@ -165,4 +168,4 @@ class HentaifoundryImageExtractor(Extractor):
            "title": text.unescape(title),
        }
        text.nameext_from_url(url, data)
-        return "http://pictures.hentai-foundry.com" + url, data
+        return "https://pictures.hentai-foundry.com" + url, data
--- a/gallery_dl/extractor/imgchili.py
+++ b/gallery_dl/extractor/imgchili.py
@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-

-# Copyright 2014-2016 Mike Fährmann
+# Copyright 2014-2017 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.

-"""Extract images from http://imgchili.net/"""
+"""Extract images from https://imgchili.net/"""

 from .common import Extractor, Message
 from .. import text
@ -16,7 +16,7 @@ class ImgchiliExtractor(Extractor):
    """Base class for imgchili extractors"""
    category = "imgchili"
    directory_fmt = ["{category}"]
-    url_base = "http://imgchili.net/"
+    url_base = "https://imgchili.net/"

    def __init__(self, match):
        Extractor.__init__(self)
--- a/gallery_dl/extractor/khinsider.py
+++ b/gallery_dl/extractor/khinsider.py
@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-

-# Copyright 2016 Mike Fährmann
+# Copyright 2016-2017 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.

-"""Extract soundtracks from http://khinsider.com/"""
+"""Extract soundtracks from https://downloads.khinsider.com/"""

 from .common import AsynchronousExtractor, Message
 from .. import text, exception
@ -31,7 +31,7 @@ class KhinsiderSoundtrackExtractor(AsynchronousExtractor):
        self.album = match.group(1)

    def items(self):
-        url = ("http://downloads.khinsider.com/game-soundtracks/album/" +
+        url = ("https://downloads.khinsider.com/game-soundtracks/album/" +
               self.album)
        page = self.request(url, encoding="utf-8").text
        data = self.get_job_metadata(page)
--- a/gallery_dl/extractor/nhentai.py
+++ b/gallery_dl/extractor/nhentai.py
@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-

-# Copyright 2015 Mike Fährmann
+# Copyright 2015-2017 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.

-"""Extract images from http://nhentai.net/"""
+"""Extract images from https://nhentai.net/"""

 from .common import Extractor, Message
 from .. import text
@ -46,7 +46,7 @@ class NhentaiGalleryExtractor(Extractor):

    def get_gallery_info(self):
        """Extract and return gallery-info"""
-        page = self.request("http://nhentai.net/g/" + self.gid + "/1/").text
+        page = self.request("https://nhentai.net/g/" + self.gid + "/1/").text
        media_url, pos = text.extract(
            page, ".reader({\n\t\t\tmedia_url: '", "'")
        json_data, pos = text.extract(
--- a/gallery_dl/extractor/pixiv.py
+++ b/gallery_dl/extractor/pixiv.py
@ -6,7 +6,7 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.

-"""Extract images and ugoira from http://www.pixiv.net/"""
+"""Extract images and ugoira from https://www.pixiv.net/"""

 from .common import Extractor, Message
 from .. import config, text, exception
@ -31,8 +31,8 @@ class PixivUserExtractor(Extractor):
            "exception": exception.NotFoundError,
        }),
    ]
-    member_url = "http://www.pixiv.net/member_illust.php"
-    illust_url = "http://www.pixiv.net/member_illust.php?mode=medium"
+    member_url = "https://www.pixiv.net/member_illust.php"
+    illust_url = "https://www.pixiv.net/member_illust.php?mode=medium"

    def __init__(self, match):
        Extractor.__init__(self)
--- a/gallery_dl/extractor/powermanga.py
+++ b/gallery_dl/extractor/powermanga.py
@ -6,7 +6,7 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.

-"""Extractors for http://powermanga.org/"""
+"""Extractors for https://powermanga.org/"""

 from . import foolslide

--- a/gallery_dl/extractor/safebooru.py
+++ b/gallery_dl/extractor/safebooru.py
@ -6,7 +6,7 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.

-"""Extract images from http://safebooru.org/"""
+"""Extract images from https://safebooru.org/"""

 from . import booru

@ -14,7 +14,7 @@ from . import booru
 class SafebooruExtractor(booru.XMLBooruExtractor):
    """Base class for safebooru extractors"""
    category = "safebooru"
-    api_url = "http://safebooru.org/index.php"
+    api_url = "https://safebooru.org/index.php"
    pagestart = 0
    pagekey = "pid"

@ -28,7 +28,7 @@ class SafebooruTagExtractor(SafebooruExtractor, booru.BooruTagExtractor):
    pattern = [(r"(?:https?://)?(?:www\.)?safebooru\.org/(?:index\.php)?"
                r"\?page=post&s=list&tags=([^&]+)")]
    test = [("http://safebooru.org/index.php?page=post&s=list&tags=bonocho", {
-        "url": "c91e04ffbdf317fae95b2e160c8345503d9fb730",
+        "url": "17c61b386530cf4c30842c9f580d15ef1cd09586",
        "content": "e5ad4c5bf241b1def154958535bef6c2f6b733eb",
    })]

@ -39,6 +39,6 @@ class SafebooruPostExtractor(SafebooruExtractor, booru.BooruPostExtractor):
    pattern = [(r"(?:https?://)?(?:www\.)?safebooru\.org/(?:index\.php)?"
                r"\?page=post&s=view&id=(\d+)")]
    test = [("http://safebooru.org/index.php?page=post&s=view&id=1169132", {
-        "url": "bcb6047665729c7c9db243a27f41cbef9af1ecef",
+        "url": "cf05e37a3c62b2d55788e2080b8eabedb00f999b",
        "content": "93b293b27dabd198afafabbaf87c49863ac82f27",
    })]
--- a/gallery_dl/job.py
+++ b/gallery_dl/job.py
@ -61,7 +61,6 @@ class Job():
                      exc.__class__.__name__, err)
            log.debug("Traceback", exc_info=True)

-
    def dispatch(self, msg):
        """Call the appropriate message handler"""
        if msg[0] == Message.Url:
@ -297,7 +296,7 @@ class DataJob(Job):
                    copy = [
                        part.copy() if hasattr(part, "copy") else part
                        for part in msg
-                ]
+                    ]
                self.data.append(copy)
        except Exception as exc:
            self.data.append((exc.__class__.__name__, str(exc)))
--- a/test/test_extractors.py
+++ b/test/test_extractors.py
@ -51,7 +51,7 @@ skip = [
    # dont work on travis-ci
    "exhentai", "kissmanga", "mangafox", "dynastyscans",
    # temporary issues
-    "mangashare", "readcomics",
+    "mangashare", "readcomics", "pawoo",
 ]
 # enable selective testing for direct calls
 if __name__ == '__main__' and len(sys.argv) > 1: