mirror_maker
/
gallery-dl
mirror of https://github.com/mikf/gallery-dl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improve supportedsites.rst and build script

Browse Source
pull/13/head
Mike Fährmann 8 years ago
parent 13dc5d72bc
commit 9f32cf1f4e
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
3 changed files with 295 additions and 102 deletions
Show Stats Download Patch File Download Diff File

2
README.rst
Unescape View File

@ -87,7 +87,7 @@ Supported Sites
* exhentai.org
* nhentai.net
* luscious.net
* hentaifoundry.com
* hentai-foundry.com
* deviantart.com
* tumblr.com
* `Complete List`_

191
scripts/build_supportedsites.py
Unescape View File

@ -2,35 +2,186 @@
import sys
import os.path
import urllib.parse
ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.realpath(ROOTDIR))
import gallery_dl.extractor
categories = {}
skip = ["test", "recursive"]
for extr in gallery_dl.extractor.extractors():
if extr.category in skip:
continue
CATEGORY_MAP = {
"deviantart" : "DeviantArt",
"dokireader" : "Doki Reader",
"dynastyscans" : "Dynasty Reader",
"e621" : "e621",
"exhentai" : "ExHentai",
"fallenangels" : "Fallen Angels Scans",
"gomanga" : "GoManga",
"hbrowse" : "HBrowse",
"hentai2read" : "Hentai2Read",
"hentaifoundry" : "Hentai Foundry",
"hentaihere" : "HentaiHere",
"hitomi" : "Hitomi.la",
"imagebam" : "ImageBam",
"imagefap" : "ImageFap",
"imgbox" : "imgbox",
"imgchili" : "imgChili",
"imgth" : "imgth",
"imgur" : "imgur",
"jaiminisbox" : "Jaimini's Box",
"kireicake" : "Kirei Cake",
"kisscomic" : "KissComic",
"kissmanga" : "KissManga",
"mangafox" : "Manga Fox",
"mangahere" : "Manga Here",
"mangapark" : "MangaPark",
"mangastream" : "Manga Stream",
"nhentai" : "nhentai",
"nijie" : "nijie",
"powermanga" : "PowerManga",
"readcomiconline": "Read Comic Online",
"rule34" : "Rule 34",
"sankaku" : "Sankaku Channel",
"seaotterscans" : "Sea Otter Scans",
"seiga" : "Niconico Seiga",
"senmanga" : "Sen Manga",
"sensescans" : "Sense-Scans",
"spectrumnexus" : "Spectrum Nexus",
"worldthree" : "World Three",
"yomanga" : "YoManga",
"yonkouprod" : "Yonkou Productions",
}
SUBCATEGORY_MAP = {
"account": "Images from Users",
"gallery": "Galleries",
"image" : "individual Images",
"issue" : "Comic-Issues",
"manga" : "Manga",
"status" : "Images from Statuses",
"tag" : "Tag-Searches",
"user" : "Images from Users",
"work" : "Individual Images",
}
class RstColumn():
def __init__(self, title, data):
self.title = title
self.data = self._transform(data)
self.size = max(len(value) for value in data + [title])
self.title = self._pad(self.title)
for i, value in enumerate(self.data):
self.data[i] = self._pad(value)
def __str__(self):
return self.title
def __len__(self):
return len(self.data)
def __getitem__(self, key):
return self.data[key] if key < len(self.data) else [""]
def _transform(self, data):
return [
value if isinstance(value, str) else ", ".join(value)
for value in data
]
def _pad(self, s):
return s + " " * (self.size - len(s))
class RstTable():
def __init__(self, columns):
self.columns = columns
self.rowcount = max(len(col) for col in columns)
self.sep = "+" + "+".join("-" * col.size for col in columns) + "+"
def __iter__(self):
yield self.sep
yield "|" + "|".join(col.title for col in self.columns) + "|"
yield self.sep.replace("-", "=")
for i in range(self.rowcount):
yield self._format_row(i)
yield self.sep
def _format_row(self, row):
return "|" + "|".join(col[row] for col in self.columns) + "|"
def build_list():
exts = []
sub = []
last = None
for ex in gallery_dl.extractor.extractors():
c, sc = ex.category, [ex.subcategory, ex]
if c == last or not last:
sub.append(sc)
elif last:
if sub[0][0] and not (len(sub) == 1 and sub[0][0] == "image"):
sub.sort(key=sub_keys)
exts.append([last, sub])
sub = [sc]
last = c
exts.append([last, sorted(sub)])
for ext in exts:
ext[0] = map_category(ext[0])
for sub in ext[1]:
sub[0] = map_subcategory(sub[0])
exts.sort(key=lambda x: x[0].lower())
return exts
def get_domain(classes):
try:
categories[extr.category]["sc"].append(extr.subcategory)
except KeyError:
url = extr.__doc__.split()[-1]
if "." not in url[-5:-2]:
url = sys.modules[extr.__module__].__doc__.split()[-1]
url = urllib.parse.urlsplit(url).netloc
if url.startswith("www."):
url = url[4:]
categories[extr.category] = {
"url": url,
"sc": [extr.subcategory],
}
url = sys.modules[classes[0].__module__].__doc__.split()[-1]
if url.startswith("http"):
return url
except (IndexError, AttributeError):
pass
return ""
def map_category(c):
return CATEGORY_MAP.get(c, c.capitalize())
def map_subcategory(sc):
return SUBCATEGORY_MAP.get(sc, sc.capitalize() + "s")
def sub_keys(sc):
if sc[0] == "user":
return "A"
return sc[0]
exts = build_list()
columns = [
RstColumn("Site", [
ext[0]
for ext in exts
]),
RstColumn("URL", [
get_domain([subc[1] for subc in ext[1]])
for ext in exts
]),
RstColumn("Capabilities", [
", ".join(subc[0] for subc in ext[1])
for ext in exts
]),
]
outfile = sys.argv[1] if len(sys.argv) > 1 else "supportedsites.rst"
with open(os.path.join(ROOTDIR, outfile), "w") as file:
file.write("Supported Sites\n"
"===============\n")
for info in sorted(categories.values(), key=lambda x: x["url"]):
file.write("- " + info["url"] + "\n")
for line in RstTable(columns):
file.write(line + "\n")

204
supportedsites.rst
Unescape View File

@ -1,83 +1,125 @@
Supported Sites
===============
- 4chan.org
- 8ch.net
- bato.to
- behoimi.org
- chan.sankakucomplex.com
- chronos.to
- coreimg.net
- danbooru.donmai.us
- deviantart.com
- dynasty-scans.com
- e621.net
- exhentai.org
- fapat.me
- fascans.com
- gelbooru.com
- gomanga.co
- hbrowse.com
- hentai2read.com
- hentaifoundry.com
- hentaihere.com
- hitomi.la
- hosturimage.com
- imagebam.com
- imagefap.com
- imageontime.org
- imagetwist.com
- imagevenue.com
- img.yt
- img4ever.net
- imgbox.com
- imgcandy.net
- imgchili.net
- imgmaid.net
- imgspice.com
- imgspot.org
- imgth.com
- imgtrex.com
- imgtrial.com
- imgupload.yt
- imgur.com
- jaiminisbox.com
- khinsider.com
- kisscomic.us
- kissmanga.com
- kobato.hologfx.com
- konachan.com
- luscious.net
- mangafox.me
- mangahere.co
- mangapanda.com
- mangapark.me
- mangareader.net
- mangashare.com
- mangastream.com
- nhentai.net
- nijie.info
- pawoo.net
- pic-maniac.com
- pinterest.com
- pixhost.org
- pixiv.net
- postimg.org
- powermanga.org
- rapidimg.net
- raw.senmanga.com
- readcomiconline.to
- readcomics.tv
- reader.kireicake.com
- reader.seaotterscans.com
- rule34.xxx
- safebooru.org
- seiga.nicovideo.jp
- sensescans.com
- slide.world-three.org
- thespectrum.net
- tumblr.com
- turboimagehost.com
- twitter.com
- yande.re
- yomanga.co
- yonkouprod.com
+-------------------+---------------------------------------+----------------------------------------------------------+
|Site |URL |Capabilities |
+===================+=======================================+==========================================================+
|3dbooru |http://behoimi.org/ |Pools, Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|4chan |https://www.4chan.org/ |Threads |
+-------------------+---------------------------------------+----------------------------------------------------------+
|8chan |https://8ch.net/ |Threads |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Batoto |https://bato.to/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Danbooru |https://danbooru.donmai.us/ |Pools, Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|DeviantArt |https://www.deviantart.com/ |Images from Users, Favorites, individual Images |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Doki Reader |https://kobato.hologfx.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Dynasty Reader |https://dynasty-scans.com/ |Chapters |
+-------------------+---------------------------------------+----------------------------------------------------------+
|e621 |https://e621.net/ |Pools, Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|ExHentai |https://exhentai.org/ |Galleries |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Fallen Angels Scans|https://fascans.com/ |Chapters |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Gelbooru |https://gelbooru.com/ |Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|GoManga |https://gomanga.co/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|HBrowse |http://www.hbrowse.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Hentai Foundry |https://www.hentai-foundry.com/ |Images from Users, individual Images |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Hentai2Read |https://hentai2read.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|HentaiHere |https://hentaihere.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Hitomi.la |https://hitomi.la/ |Galleries |
+-------------------+---------------------------------------+----------------------------------------------------------+
|ImageBam |http://www.imagebam.com/ |Galleries, individual Images |
+-------------------+---------------------------------------+----------------------------------------------------------+
|ImageFap |http://imagefap.com/ |Images from Users, Galleries, individual Images |
+-------------------+---------------------------------------+----------------------------------------------------------+
|imgbox |https://imgbox.com/ |Galleries, individual Images |
+-------------------+---------------------------------------+----------------------------------------------------------+
|imgChili |https://imgchili.net/ |Albums, individual Images |
+-------------------+---------------------------------------+----------------------------------------------------------+
|imgth |https://imgth.com/ |Galleries |
+-------------------+---------------------------------------+----------------------------------------------------------+
|imgur |https://imgur.com/ |Albums |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Jaimini's Box |https://jaiminisbox.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Khinsider |https://downloads.khinsider.com/ |Soundtracks |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Kirei Cake |https://reader.kireicake.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|KissComic |http://kisscomic.us/ |Comics, Comic-Issues |
+-------------------+---------------------------------------+----------------------------------------------------------+
|KissManga |http://kissmanga.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Konachan |https://konachan.com/ |Pools, Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Luscious |https://luscious.net/ |Albums |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Manga Fox |http://www.mangafox.me/ |Chapters |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Manga Here |http://www.mangahere.co/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Manga Stream |https://mangastream.com/ |Chapters |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Mangapanda |http://www.mangapanda.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|MangaPark |http://mangapark.me/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Mangareader |http://www.mangareader.net/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Mangashare |http://www.mangashare.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|nhentai |https://nhentai.net/ |Galleries |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Niconico Seiga |http://seiga.nicovideo.jp |Images from Users, individual Images |
+-------------------+---------------------------------------+----------------------------------------------------------+
|nijie |https://nijie.info/ |Images from Users, individual Images |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Pawoo |https://pawoo.net |Images from Users, Images from Statuses |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Pinterest |https://www.pinterest.com |Boards, Pins, Pinits |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Pixiv |https://www.pixiv.net/ |Images from Users, Bookmarks, Favorites, Individual Images|
+-------------------+---------------------------------------+----------------------------------------------------------+
|PowerManga |https://powermanga.org/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Read Comic Online |http://readcomiconline.to/ |Comics, Comic-Issues |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Readcomics |http://readcomics.tv/ |Comics, Comic-Issues |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Rule 34 |https://rule34.xxx/ |Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Safebooru |https://safebooru.org/ |Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Sankaku Channel |https://chan.sankakucomplex.com/ |Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Sea Otter Scans |https://reader.seaotterscans.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Sen Manga |http://raw.senmanga.com/ |Chapters |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Sense-Scans |http://sensescans.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Spectrum Nexus |http://www.thespectrum.net/manga_scans/|Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Tumblr |https://www.tumblr.com/ |Images from Users, Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Twitter |https://twitter.com/ |Tweets |
+-------------------+---------------------------------------+----------------------------------------------------------+
|World Three |http://www.slide.world-three.org/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Yandere |https://yande.re/ |Pools, Posts, Tag-Searches |
+-------------------+---------------------------------------+----------------------------------------------------------+
|YoManga |https://yomanga.co/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+
|Yonkou Productions |https://yonkouprod.com/ |Chapters, Manga |
+-------------------+---------------------------------------+----------------------------------------------------------+

Write Preview
Loading…
Cancel
Save