diff --git a/README.rst b/README.rst index e292cec5..2e5fee84 100644 --- a/README.rst +++ b/README.rst @@ -87,7 +87,7 @@ Supported Sites * exhentai.org * nhentai.net * luscious.net -* hentaifoundry.com +* hentai-foundry.com * deviantart.com * tumblr.com * `Complete List`_ diff --git a/scripts/build_supportedsites.py b/scripts/build_supportedsites.py index 7228a207..168968ca 100755 --- a/scripts/build_supportedsites.py +++ b/scripts/build_supportedsites.py @@ -2,35 +2,186 @@ import sys import os.path -import urllib.parse ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.realpath(ROOTDIR)) import gallery_dl.extractor -categories = {} -skip = ["test", "recursive"] -for extr in gallery_dl.extractor.extractors(): - if extr.category in skip: - continue +CATEGORY_MAP = { + "deviantart" : "DeviantArt", + "dokireader" : "Doki Reader", + "dynastyscans" : "Dynasty Reader", + "e621" : "e621", + "exhentai" : "ExHentai", + "fallenangels" : "Fallen Angels Scans", + "gomanga" : "GoManga", + "hbrowse" : "HBrowse", + "hentai2read" : "Hentai2Read", + "hentaifoundry" : "Hentai Foundry", + "hentaihere" : "HentaiHere", + "hitomi" : "Hitomi.la", + "imagebam" : "ImageBam", + "imagefap" : "ImageFap", + "imgbox" : "imgbox", + "imgchili" : "imgChili", + "imgth" : "imgth", + "imgur" : "imgur", + "jaiminisbox" : "Jaimini's Box", + "kireicake" : "Kirei Cake", + "kisscomic" : "KissComic", + "kissmanga" : "KissManga", + "mangafox" : "Manga Fox", + "mangahere" : "Manga Here", + "mangapark" : "MangaPark", + "mangastream" : "Manga Stream", + "nhentai" : "nhentai", + "nijie" : "nijie", + "powermanga" : "PowerManga", + "readcomiconline": "Read Comic Online", + "rule34" : "Rule 34", + "sankaku" : "Sankaku Channel", + "seaotterscans" : "Sea Otter Scans", + "seiga" : "Niconico Seiga", + "senmanga" : "Sen Manga", + "sensescans" : "Sense-Scans", + "spectrumnexus" : "Spectrum Nexus", + "worldthree" : "World Three", + "yomanga" : "YoManga", + "yonkouprod" : "Yonkou Productions", + +} + +SUBCATEGORY_MAP = { + "account": "Images from Users", + "gallery": "Galleries", + "image" : "individual Images", + "issue" : "Comic-Issues", + "manga" : "Manga", + "status" : "Images from Statuses", + "tag" : "Tag-Searches", + "user" : "Images from Users", + "work" : "Individual Images", +} + + +class RstColumn(): + + def __init__(self, title, data): + self.title = title + self.data = self._transform(data) + self.size = max(len(value) for value in data + [title]) + + self.title = self._pad(self.title) + for i, value in enumerate(self.data): + self.data[i] = self._pad(value) + + def __str__(self): + return self.title + + def __len__(self): + return len(self.data) + + def __getitem__(self, key): + return self.data[key] if key < len(self.data) else [""] + + def _transform(self, data): + return [ + value if isinstance(value, str) else ", ".join(value) + for value in data + ] + + def _pad(self, s): + return s + " " * (self.size - len(s)) + + +class RstTable(): + + def __init__(self, columns): + self.columns = columns + self.rowcount = max(len(col) for col in columns) + self.sep = "+" + "+".join("-" * col.size for col in columns) + "+" + + def __iter__(self): + yield self.sep + yield "|" + "|".join(col.title for col in self.columns) + "|" + yield self.sep.replace("-", "=") + for i in range(self.rowcount): + yield self._format_row(i) + yield self.sep + + def _format_row(self, row): + return "|" + "|".join(col[row] for col in self.columns) + "|" + + +def build_list(): + exts = [] + sub = [] + last = None + for ex in gallery_dl.extractor.extractors(): + c, sc = ex.category, [ex.subcategory, ex] + if c == last or not last: + sub.append(sc) + elif last: + if sub[0][0] and not (len(sub) == 1 and sub[0][0] == "image"): + sub.sort(key=sub_keys) + exts.append([last, sub]) + sub = [sc] + last = c + exts.append([last, sorted(sub)]) + + for ext in exts: + ext[0] = map_category(ext[0]) + for sub in ext[1]: + sub[0] = map_subcategory(sub[0]) + exts.sort(key=lambda x: x[0].lower()) + + return exts + + +def get_domain(classes): try: - categories[extr.category]["sc"].append(extr.subcategory) - except KeyError: - url = extr.__doc__.split()[-1] - if "." not in url[-5:-2]: - url = sys.modules[extr.__module__].__doc__.split()[-1] - url = urllib.parse.urlsplit(url).netloc - if url.startswith("www."): - url = url[4:] - categories[extr.category] = { - "url": url, - "sc": [extr.subcategory], - } + url = sys.modules[classes[0].__module__].__doc__.split()[-1] + if url.startswith("http"): + return url + except (IndexError, AttributeError): + pass + return "" + + +def map_category(c): + return CATEGORY_MAP.get(c, c.capitalize()) + + +def map_subcategory(sc): + return SUBCATEGORY_MAP.get(sc, sc.capitalize() + "s") + + +def sub_keys(sc): + if sc[0] == "user": + return "A" + return sc[0] + + +exts = build_list() +columns = [ + RstColumn("Site", [ + ext[0] + for ext in exts + ]), + RstColumn("URL", [ + get_domain([subc[1] for subc in ext[1]]) + for ext in exts + ]), + RstColumn("Capabilities", [ + ", ".join(subc[0] for subc in ext[1]) + for ext in exts + ]), +] outfile = sys.argv[1] if len(sys.argv) > 1 else "supportedsites.rst" with open(os.path.join(ROOTDIR, outfile), "w") as file: file.write("Supported Sites\n" "===============\n") - for info in sorted(categories.values(), key=lambda x: x["url"]): - file.write("- " + info["url"] + "\n") + for line in RstTable(columns): + file.write(line + "\n") diff --git a/supportedsites.rst b/supportedsites.rst index 6dad94e7..dfe372ce 100644 --- a/supportedsites.rst +++ b/supportedsites.rst @@ -1,83 +1,125 @@ Supported Sites =============== -- 4chan.org -- 8ch.net -- bato.to -- behoimi.org -- chan.sankakucomplex.com -- chronos.to -- coreimg.net -- danbooru.donmai.us -- deviantart.com -- dynasty-scans.com -- e621.net -- exhentai.org -- fapat.me -- fascans.com -- gelbooru.com -- gomanga.co -- hbrowse.com -- hentai2read.com -- hentaifoundry.com -- hentaihere.com -- hitomi.la -- hosturimage.com -- imagebam.com -- imagefap.com -- imageontime.org -- imagetwist.com -- imagevenue.com -- img.yt -- img4ever.net -- imgbox.com -- imgcandy.net -- imgchili.net -- imgmaid.net -- imgspice.com -- imgspot.org -- imgth.com -- imgtrex.com -- imgtrial.com -- imgupload.yt -- imgur.com -- jaiminisbox.com -- khinsider.com -- kisscomic.us -- kissmanga.com -- kobato.hologfx.com -- konachan.com -- luscious.net -- mangafox.me -- mangahere.co -- mangapanda.com -- mangapark.me -- mangareader.net -- mangashare.com -- mangastream.com -- nhentai.net -- nijie.info -- pawoo.net -- pic-maniac.com -- pinterest.com -- pixhost.org -- pixiv.net -- postimg.org -- powermanga.org -- rapidimg.net -- raw.senmanga.com -- readcomiconline.to -- readcomics.tv -- reader.kireicake.com -- reader.seaotterscans.com -- rule34.xxx -- safebooru.org -- seiga.nicovideo.jp -- sensescans.com -- slide.world-three.org -- thespectrum.net -- tumblr.com -- turboimagehost.com -- twitter.com -- yande.re -- yomanga.co -- yonkouprod.com ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Site |URL |Capabilities | ++===================+=======================================+==========================================================+ +|3dbooru |http://behoimi.org/ |Pools, Posts, Tag-Searches | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|4chan |https://www.4chan.org/ |Threads | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|8chan |https://8ch.net/ |Threads | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Batoto |https://bato.to/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Danbooru |https://danbooru.donmai.us/ |Pools, Posts, Tag-Searches | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|DeviantArt |https://www.deviantart.com/ |Images from Users, Favorites, individual Images | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Doki Reader |https://kobato.hologfx.com/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Dynasty Reader |https://dynasty-scans.com/ |Chapters | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|e621 |https://e621.net/ |Pools, Posts, Tag-Searches | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|ExHentai |https://exhentai.org/ |Galleries | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Fallen Angels Scans|https://fascans.com/ |Chapters | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Gelbooru |https://gelbooru.com/ |Posts, Tag-Searches | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|GoManga |https://gomanga.co/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|HBrowse |http://www.hbrowse.com/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Hentai Foundry |https://www.hentai-foundry.com/ |Images from Users, individual Images | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Hentai2Read |https://hentai2read.com/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|HentaiHere |https://hentaihere.com/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Hitomi.la |https://hitomi.la/ |Galleries | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|ImageBam |http://www.imagebam.com/ |Galleries, individual Images | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|ImageFap |http://imagefap.com/ |Images from Users, Galleries, individual Images | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|imgbox |https://imgbox.com/ |Galleries, individual Images | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|imgChili |https://imgchili.net/ |Albums, individual Images | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|imgth |https://imgth.com/ |Galleries | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|imgur |https://imgur.com/ |Albums | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Jaimini's Box |https://jaiminisbox.com/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Khinsider |https://downloads.khinsider.com/ |Soundtracks | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Kirei Cake |https://reader.kireicake.com/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|KissComic |http://kisscomic.us/ |Comics, Comic-Issues | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|KissManga |http://kissmanga.com/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Konachan |https://konachan.com/ |Pools, Posts, Tag-Searches | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Luscious |https://luscious.net/ |Albums | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Manga Fox |http://www.mangafox.me/ |Chapters | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Manga Here |http://www.mangahere.co/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Manga Stream |https://mangastream.com/ |Chapters | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Mangapanda |http://www.mangapanda.com/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|MangaPark |http://mangapark.me/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Mangareader |http://www.mangareader.net/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Mangashare |http://www.mangashare.com/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|nhentai |https://nhentai.net/ |Galleries | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Niconico Seiga |http://seiga.nicovideo.jp |Images from Users, individual Images | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|nijie |https://nijie.info/ |Images from Users, individual Images | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Pawoo |https://pawoo.net |Images from Users, Images from Statuses | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Pinterest |https://www.pinterest.com |Boards, Pins, Pinits | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Pixiv |https://www.pixiv.net/ |Images from Users, Bookmarks, Favorites, Individual Images| ++-------------------+---------------------------------------+----------------------------------------------------------+ +|PowerManga |https://powermanga.org/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Read Comic Online |http://readcomiconline.to/ |Comics, Comic-Issues | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Readcomics |http://readcomics.tv/ |Comics, Comic-Issues | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Rule 34 |https://rule34.xxx/ |Posts, Tag-Searches | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Safebooru |https://safebooru.org/ |Posts, Tag-Searches | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Sankaku Channel |https://chan.sankakucomplex.com/ |Tag-Searches | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Sea Otter Scans |https://reader.seaotterscans.com/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Sen Manga |http://raw.senmanga.com/ |Chapters | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Sense-Scans |http://sensescans.com/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Spectrum Nexus |http://www.thespectrum.net/manga_scans/|Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Tumblr |https://www.tumblr.com/ |Images from Users, Posts, Tag-Searches | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Twitter |https://twitter.com/ |Tweets | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|World Three |http://www.slide.world-three.org/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Yandere |https://yande.re/ |Pools, Posts, Tag-Searches | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|YoManga |https://yomanga.co/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+ +|Yonkou Productions |https://yonkouprod.com/ |Chapters, Manga | ++-------------------+---------------------------------------+----------------------------------------------------------+