diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 9252b3e0..18b9c647 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -59,7 +59,7 @@ "morecomments": false, "date-min": 0, "date-max": 253402210800, - "date-format": "", + "date-format": "%Y-%m-%dT%H:%M:%S", "id-min": "0", "id-max": "ZIK0ZJ", "recursion": 0 diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index b2f5bf73..cf5a62d0 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -1,167 +1,100 @@ Supported Sites =============== -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Site |URL |Capabilities |Authentication | -+===================+=======================================+==========================================================================================+================+ -|3dbooru |http://behoimi.org/ |Pools, Posts, Tag-Searches | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|4chan |https://www.4chan.org/ |Threads | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|8chan |https://8ch.net/ |Threads | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Batoto |https://bato.to/ |Chapters, Manga |Optional | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Danbooru |https://danbooru.donmai.us/ |Pools, Posts, Tag-Searches | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|DeviantArt |https://www.deviantart.com/ |Deviations, Favorites, Galleries, Journals | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Doki Reader |https://kobato.hologfx.com/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Dynasty Reader |https://dynasty-scans.com/ |Chapters | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|e621 |https://e621.net/ |Pools, Posts, Tag-Searches | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|ExHentai |https://exhentai.org/ |Galleries |Optional | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Fallen Angels Scans|https://www.fascans.com/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Flickr |https://www.flickr.com/ |Images from Users, Albums, Favorites, Galleries, Groups, individual Images, Search Results|Optional (OAuth)| -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Gelbooru |https://gelbooru.com/ |Posts, Tag-Searches | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Gfycat |https://gfycat.com/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|GoManga |https://gomanga.co/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|HBrowse |http://www.hbrowse.com/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Hentai Foundry |https://www.hentai-foundry.com/ |Images from Users, individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Hentai2Read |https://hentai2read.com/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|HentaiHere |https://hentaihere.com/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Hitomi.la |https://hitomi.la/ |Galleries | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|ImageBam |http://www.imagebam.com/ |Galleries, individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|ImageFap |http://imagefap.com/ |Images from Users, Galleries, individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|imgbox |https://imgbox.com/ |Galleries, individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|imgChili |https://imgchili.net/ |Albums, individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|imgth |https://imgth.com/ |Galleries | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|imgur |https://imgur.com/ |Albums, individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Jaimini's Box |https://jaiminisbox.com/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Khinsider |https://downloads.khinsider.com/ |Soundtracks | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Kirei Cake |https://reader.kireicake.com/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|KissComic |http://kisscomic.us/ |Comic-Issues, Comics | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|KissManga |http://kissmanga.com/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Konachan |https://konachan.com/ |Pools, Posts, Tag-Searches | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Luscious |https://luscious.net/ |Albums | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Manga Fox |http://www.mangafox.me/ |Chapters | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Manga Here |http://www.mangahere.co/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Manga Stream |https://mangastream.com/ |Chapters | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Mangapanda |http://www.mangapanda.com/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|MangaPark |http://mangapark.me/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Mangareader |http://www.mangareader.net/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|nhentai |https://nhentai.net/ |Galleries | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Niconico Seiga |http://seiga.nicovideo.jp |Images from Users, individual Images |Required | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|nijie |https://nijie.info/ |Images from Users, individual Images |Required | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Pawoo |https://pawoo.net |Images from Users, Images from Statuses | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Pinterest |https://www.pinterest.com |Boards, Pins, pin.it Links | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Pixiv |https://www.pixiv.net/ |Images from Users, Bookmarks, Favorites, pixiv.me Links, Individual Images |Required | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|PowerManga |https://powermanga.org/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Read Comic Online |http://readcomiconline.to/ |Comic-Issues, Comics | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Reddit |https://reddit.com/ |Submissions, Subreddits |Optional (OAuth)| -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Rule 34 |https://rule34.xxx/ |Posts, Tag-Searches | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Safebooru |https://safebooru.org/ |Posts, Tag-Searches | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Sankaku Channel |https://chan.sankakucomplex.com/ |Tag-Searches | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Sea Otter Scans |https://reader.seaotterscans.com/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Sen Manga |http://raw.senmanga.com/ |Chapters | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Sense-Scans |http://sensescans.com/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Spectrum Nexus |http://www.thespectrum.net/manga_scans/|Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Tumblr |https://www.tumblr.com/ |Images from Users, Posts, Tag-Searches | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Twitter |https://twitter.com/ |Tweets | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|World Three |http://www.slide.world-three.org/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Yandere |https://yande.re/ |Pools, Posts, Tag-Searches | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Yonkou Productions |https://yonkouprod.com/ |Chapters, Manga | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Chronos |http://chronos.to/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Coreimg |http://coreimg.net/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Fapat |http://fapat.me/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Hosturimage |https://hosturimage.com/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Imageontime |http://imageontime.org/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Imagetwist |https://imagetwist.com/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Imagevenue |http://imagevenue.com/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Img4ever |https://img4ever.net/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Imgcandy |http://imgcandy.net/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Imgmaid |https://imgmaid.net/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Imgspice |https://imgspice.com/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Imgspot |http://imgspot.org/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Imgtrex |http://imgtrex.com/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Imgtrial |http://imgtrial.com/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Imgupload |https://imgupload.yt/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Imgyt |https://img.yt/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Picmaniac |http://pic-maniac.com/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Pixhost |https://pixhost.org/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Postimg |https://postimg.org/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Rapidimg |http://rapidimg.net/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ -|Turboimagehost |http://turboimagehost.com/ |individual Images | | -+-------------------+---------------------------------------+------------------------------------------------------------------------------------------+----------------+ +==================== =================================== ================================================== ================ +Site URL Capabilities Authentication +==================== =================================== ================================================== ================ +3dbooru http://behoimi.org/ Pools, Posts, Tag-Searches +4chan https://www.4chan.org/ Threads +4plebs https://archive.4plebs.org/ Threads +8chan https://8ch.net/ Threads +Archive of Sins https://archiveofsins.com/ Threads +Archived.Moe https://archived.moe/ Threads +Batoto https://bato.to/ Chapters, Manga Optional +Danbooru https://danbooru.donmai.us/ Pools, Posts, Tag-Searches +Desuarchive https://desuarchive.org/ Threads +DeviantArt https://www.deviantart.com/ |Collections, De-1| +Doki Reader https://kobato.hologfx.com/ Chapters, Manga +Dynasty Reader https://dynasty-scans.com/ Chapters +e621 https://e621.net/ Pools, Posts, Tag-Searches +ExHentai https://exhentai.org/ Galleries Optional +Fallen Angels Scans https://www.fascans.com/ Chapters, Manga +Fireden https://boards.fireden.net/ Threads +Flickr https://www.flickr.com/ |Images from Use-2| Optional (OAuth) +Futaba Channel https://www.2chan.net/ Threads +Gelbooru https://gelbooru.com/ Posts, Tag-Searches +Gfycat https://gfycat.com/ individual Images +GoManga https://gomanga.co/ Chapters, Manga +HBrowse http://www.hbrowse.com/ Chapters, Manga +Hentai Foundry https://www.hentai-foundry.com/ Images from Users, individual Images +Hentai2Read https://hentai2read.com/ Chapters, Manga +HentaiHere https://hentaihere.com/ Chapters, Manga +Hitomi.la https://hitomi.la/ Galleries +ImageBam http://www.imagebam.com/ Galleries, individual Images +ImageFap http://imagefap.com/ Images from Users, Galleries, individual Images +imgbox https://imgbox.com/ Galleries, individual Images +imgChili https://imgchili.net/ Albums, individual Images +imgth https://imgth.com/ Galleries +imgur https://imgur.com/ Albums, individual Images +Jaimini's Box https://jaiminisbox.com/ Chapters, Manga +Khinsider https://downloads.khinsider.com/ Soundtracks +Kirei Cake https://reader.kireicake.com/ Chapters, Manga +KissComic http://kisscomic.us/ Comic-Issues, Comics +KissManga http://kissmanga.com/ Chapters, Manga +Konachan https://konachan.com/ Pools, Posts, Tag-Searches +Love is Over Archive https://archive.loveisover.me/ Threads +Luscious https://luscious.net/ Albums +Manga Fox http://www.mangafox.me/ Chapters +Manga Here http://www.mangahere.co/ Chapters, Manga +Manga Stream https://mangastream.com/ Chapters +Mangapanda http://www.mangapanda.com/ Chapters, Manga +MangaPark http://mangapark.me/ Chapters, Manga +Mangareader http://www.mangareader.net/ Chapters, Manga +nhentai https://nhentai.net/ Galleries +Niconico Seiga http://seiga.nicovideo.jp Images from Users, individual Images Required +nijie https://nijie.info/ Images from Users, individual Images Required +Nyafuu Archive https://archive.nyafuu.org/ Threads +Pawoo https://pawoo.net Images from Users, Images from Statuses +Pinterest https://www.pinterest.com Boards, Pins, pin.it Links +Pixiv https://www.pixiv.net/ |Images from Use-3| Required +PowerManga https://powermanga.org/ Chapters, Manga +Read Comic Online http://readcomiconline.to/ Comic-Issues, Comics +Reddit https://reddit.com/ Submissions, Subreddits Optional (OAuth) +Rule 34 https://rule34.xxx/ Posts, Tag-Searches +Safebooru https://safebooru.org/ Posts, Tag-Searches +Sankaku Channel https://chan.sankakucomplex.com/ Tag-Searches +Sea Otter Scans https://reader.seaotterscans.com/ Chapters, Manga +Sen Manga http://raw.senmanga.com/ Chapters +Sense-Scans http://sensescans.com/ Chapters, Manga +Spectrum Nexus |http://www.thes-0| Chapters, Manga +Tumblr https://www.tumblr.com/ Images from Users, Posts, Tag-Searches +Twitter https://twitter.com/ Tweets +World Three http://www.slide.world-three.org/ Chapters, Manga +Yandere https://yande.re/ Pools, Posts, Tag-Searches +Yonkou Productions https://yonkouprod.com/ Chapters, Manga +Chronos http://chronos.to/ individual Images +Coreimg http://coreimg.net/ individual Images +Fapat http://fapat.me/ individual Images +Hosturimage https://hosturimage.com/ individual Images +Imageontime http://imageontime.org/ individual Images +Imagetwist https://imagetwist.com/ individual Images +Imagevenue http://imagevenue.com/ individual Images +Img4ever https://img4ever.net/ individual Images +Imgcandy http://imgcandy.net/ individual Images +Imgmaid https://imgmaid.net/ individual Images +Imgspice https://imgspice.com/ individual Images +Imgspot http://imgspot.org/ individual Images +Imgtrex http://imgtrex.com/ individual Images +Imgtrial http://imgtrial.com/ individual Images +Imgupload https://imgupload.yt/ individual Images +Imgyt https://img.yt/ individual Images +Picmaniac http://pic-maniac.com/ individual Images +Pixhost https://pixhost.org/ individual Images +Postimg https://postimg.org/ individual Images +Rapidimg http://rapidimg.net/ individual Images +Turboimagehost http://turboimagehost.com/ individual Images +==================== =================================== ================================================== ================ + +.. |http://www.thes-0| replace:: http://www.thespectrum.net/manga_scans/ +.. |Collections, De-1| replace:: Collections, Deviations, Favorites, Folders, Galleries, Journals +.. |Images from Use-2| replace:: Images from Users, Albums, Favorites, Galleries, Groups, individual Images, Search Results +.. |Images from Use-3| replace:: Images from Users, Bookmarks, Favorites, pixiv.me Links, Individual Images diff --git a/scripts/build_supportedsites.py b/scripts/build_supportedsites.py index 590dac84..8559c388 100755 --- a/scripts/build_supportedsites.py +++ b/scripts/build_supportedsites.py @@ -9,6 +9,9 @@ import gallery_dl.extractor CATEGORY_MAP = { + "2chan" : "Futaba Channel", + "archivedmoe" : "Archived.Moe", + "archiveofsins" : "Archive of Sins", "deviantart" : "DeviantArt", "dokireader" : "Doki Reader", "dynastyscans" : "Dynasty Reader", @@ -31,12 +34,14 @@ CATEGORY_MAP = { "kireicake" : "Kirei Cake", "kisscomic" : "KissComic", "kissmanga" : "KissManga", + "loveisover" : "Love is Over Archive", "mangafox" : "Manga Fox", "mangahere" : "Manga Here", "mangapark" : "MangaPark", "mangastream" : "Manga Stream", "nhentai" : "nhentai", "nijie" : "nijie", + "nyafuu" : "Nyafuu Archive", "powermanga" : "PowerManga", "readcomiconline": "Read Comic Online", "rule34" : "Rule 34", @@ -82,10 +87,13 @@ IGNORE_LIST = ( class RstColumn(): - def __init__(self, title, data): + def __init__(self, title, data, size=None): self.title = title self.data = self._transform(data) - self.size = max(len(value) for value in data + [title]) + if not size: + self.size = max(len(value) for value in data + [title]) + else: + self.size = size self.title = self._pad(self.title) for i, value in enumerate(self.data): @@ -107,7 +115,10 @@ class RstColumn(): ] def _pad(self, s): - return s + " " * (self.size - len(s)) + if len(s) <= self.size: + return s + " " * (self.size - len(s)) + else: + return substitute(s, self.size) class RstTable(): @@ -115,18 +126,26 @@ class RstTable(): def __init__(self, columns): self.columns = columns self.rowcount = max(len(col) for col in columns) - self.sep = "+" + "+".join("-" * col.size for col in columns) + "+" + self.sep = " ".join("=" * col.size for col in columns) def __iter__(self): yield self.sep - yield "|" + "|".join(col.title for col in self.columns) + "|" - yield self.sep.replace("-", "=") + yield " ".join(col.title for col in self.columns) + yield self.sep for i in range(self.rowcount): yield self._format_row(i) - yield self.sep + yield self.sep def _format_row(self, row): - return "|" + "|".join(col[row] for col in self.columns) + "|" + return " ".join(col[row] for col in self.columns) + + +_subs = [] + +def substitute(value, size): + sub = "|{}-{}|".format(value[:15], len(_subs)) + _subs.append((value, sub)) + return sub + " " * (size - len(sub)) def build_list(): @@ -196,15 +215,15 @@ columns = [ RstColumn("Site", [ extrlist[0].cat for extrlist in extractors - ]), + ], 20), RstColumn("URL", [ get_domain(extrlist) for extrlist in extractors - ]), + ], 35), RstColumn("Capabilities", [ ", ".join(extr.subcat for extr in extrlist) for extrlist in extractors - ]), + ], 50), RstColumn("Authentication", [ AUTH_MAP.get(extrlist[0].category, "") for extrlist in extractors @@ -216,4 +235,7 @@ with open(os.path.join(ROOTDIR, "docs", outfile), "w") as file: file.write("Supported Sites\n" "===============\n") for line in RstTable(columns): - file.write(line + "\n") + file.write(line.rstrip() + "\n") + file.write("\n") + for val, sub in _subs: + file.write(".. {} replace:: {}\n".format(sub, val))