mirror_maker
/
gallery-dl
mirror of https://github.com/mikf/gallery-dl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[deviantart] add support for OAuth2 authentication

Browse Source 
Some user galleries [*] require you to be either logged in or
authenticated via OAuth2 to access their deviations.

[*] e.g. https://polinaegorussia.deviantart.com/gallery/

--------------

known issue:
A deviantart 'refresh_token' can only be used once and gets updated
whenever it is used to request a new 'access_token', so storing its
initial value in a config file and reusing it again and again is not
possible.
pull/73/head
Mike Fährmann 7 years ago
parent 91c2aed077
commit fc7d165c97
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
9 changed files with 119 additions and 63 deletions
Show Stats Download Patch File Download Diff File

1
CHANGELOG.md
Unescape View File

@ -4,6 +4,7 @@
- Added support for:
- `slideshare` - https://www.slideshare.net/ ([#54](https://github.com/mikf/gallery-dl/issues/54))
- Added pool- and post-extractors for `sankaku`
- Added OAuth user authentication for `deviantart`
- Improved `luscious` to support `members.luscious.net` URLs ([#55](https://github.com/mikf/gallery-dl/issues/55))
- Fixed extraction issues for `nhentai` and `khinsider`

5
README.rst
Unescape View File

@ -190,8 +190,9 @@ or you can provide them directly via the
OAuth
-----
*gallery-dl* supports user authentication via OAuth_ for ``flickr`` and
``reddit``. This is entirely optional, but grants *gallery-dl* the ability
*gallery-dl* supports user authentication via OAuth_ for
``deviantart``, ``flickr`` and ``reddit``.
This is entirely optional, but grants *gallery-dl* the ability
to issue requests on your account's behalf and enables it to access resources
which would otherwise be unavailable to a public user.

21
docs/configuration.rst
Unescape View File

@ -271,7 +271,7 @@ Description Controls the behavior when downloading a file whose filename
extractor.*.sleep
----------------
-----------------
=========== =====
Type ``float``
Default ``0``
@ -370,6 +370,19 @@ Description Request full-sized original images if available.
=========== =====
extractor.deviantart.refresh-token
----------------------------------
=========== =====
Type ``string``
Default ``null``
Description The ``refresh_token`` value you get from linking your
DeviantArt account to *gallery-dl*.
Using a ``refresh_token`` allows you to access private or otherwise
not publicly available deviations.
=========== =====
extractor.exhentai.original
---------------------------
=========== =====
@ -576,10 +589,10 @@ extractor.reddit.refresh-token
=========== =====
Type ``string``
Default ``null``
Description The ``refresh_token`` value you get from linking your Reddit account
to *gallery-dl*.
Description The ``refresh_token`` value you get from linking your
Reddit account to *gallery-dl*.
Using the ``refresh_token`` allows you to access private or otherwise
Using a ``refresh_token`` allows you to access private or otherwise
not publicly available subreddits, given that your account is
authorized to do so,
but requests to the reddit API are going to be rate limited

1
docs/gallery-dl.conf
Unescape View File

@ -93,6 +93,7 @@
},
"deviantart":
{
"refresh-token": null,
"flat": true,
"mature": true,
"original": true

2
docs/supportedsites.rst
Unescape View File

@ -13,7 +13,7 @@ Archived.Moe https://archived.moe/ Threads
Batoto https://bato.to/ Chapters, Manga Optional
Danbooru https://danbooru.donmai.us/ Pools, Popular Images, Posts, Tag-Searches
Desuarchive https://desuarchive.org/ Threads
DeviantArt https://www.deviantart.com/ |Collections, De-1|
DeviantArt https://www.deviantart.com/ |Collections, De-1| Optional (OAuth)
Doki Reader https://kobato.hologfx.com/ Chapters, Manga
Dynasty Reader https://dynasty-scans.com/ Chapters
e621 https://e621.net/ Pools, Popular Images, Posts, Tag-Searches

38
gallery_dl/extractor/deviantart.py
Unescape View File

@ -341,18 +341,24 @@ class DeviantartJournalExtractor(DeviantartExtractor):
class DeviantartAPI():
"""Minimal interface for the deviantart API"""
def __init__(self, extractor, client_id="5388",
client_secret="76b08c69cfb27f26d6161f9ab6d061a1"):
CLIENT_ID = "5388"
CLIENT_SECRET = "76b08c69cfb27f26d6161f9ab6d061a1"
def __init__(self, extractor):
self.session = extractor.session
self.headers = {}
self.log = extractor.log
self.client_id = extractor.config("client-id", client_id)
self.client_secret = extractor.config("client-secret", client_secret)
self.headers = {}
self.delay = 0
self.mature = extractor.config("mature", "true")
if not isinstance(self.mature, str):
self.mature = "true" if self.mature else "false"
self.refresh_token = extractor.config("refresh-token")
self.client_id = extractor.config("client-id", self.CLIENT_ID)
self.client_secret = extractor.config(
"client-secret", self.CLIENT_SECRET)
def browse_user_journals(self, username, offset=0):
"""Yield all journal entries of a specific user"""
endpoint = "browse/user/journals"
@ -422,21 +428,22 @@ class DeviantartAPI():
def authenticate(self):
"""Authenticate the application by requesting an access token"""
access_token = self._authenticate_impl(
self.client_id, self.client_secret
)
access_token = self._authenticate_impl(self.refresh_token)
self.headers["Authorization"] = access_token
@cache(maxage=3590, keyarg=1)
def _authenticate_impl(self, client_id, client_secret):
def _authenticate_impl(self, refresh_token):
"""Actual authenticate implementation"""
url = "https://www.deviantart.com/oauth2/token"
data = {
"grant_type": "client_credentials",
"client_id": client_id,
"client_secret": client_secret,
}
response = self.session.post(url, data=data)
if refresh_token:
self.log.info("Refreshing access token")
data = {"grant_type": "refresh_token",
"refresh_token": refresh_token}
else:
self.log.info("Requesting public access token")
data = {"grant_type": "client_credentials"}
auth = (self.client_id, self.client_secret)
response = self.session.post(url, data=data, auth=auth)
if response.status_code != 200:
raise exception.AuthenticationError()
return "Bearer " + response.json()["access_token"]
@ -470,6 +477,7 @@ class DeviantartAPI():
try:
return response.json()
except ValueError:
self.log.error("Failed to parse API response")
return {}
def _pagination(self, endpoint, params=None):

94
gallery_dl/extractor/oauth.py
Unescape View File

@ -9,7 +9,7 @@
"""Utility classes to setup OAuth and link a users account to gallery-dl"""
from .common import Extractor, Message
from . import reddit, flickr
from . import deviantart, flickr, reddit
from .. import util
import os
import urllib.parse
@ -20,7 +20,7 @@ class OAuthBase(Extractor):
category = "oauth"
redirect_uri = "http://localhost:6414/"
def __init__(self):
def __init__(self, match):
Extractor.__init__(self)
self.client = None
@ -71,56 +71,87 @@ class OAuthBase(Extractor):
print(url, end="\n\n", flush=True)
return self.recv()
def _oauth2_authorization_code_grant(
self, client_id, client_secret, auth_url, token_url, scope):
"""Perform an OAuth2 authorization code grant"""
class OAuthReddit(OAuthBase):
subcategory = "reddit"
pattern = ["oauth:reddit$"]
def __init__(self, match):
OAuthBase.__init__(self)
self.session.headers["User-Agent"] = reddit.RedditAPI.USER_AGENT
self.client_id = reddit.RedditAPI.CLIENT_ID
self.state = "gallery-dl:{}:{}".format(
state = "gallery-dl:{}:{}".format(
self.subcategory, util.OAuthSession.nonce(8))
def items(self):
yield Message.Version, 1
url = "https://www.reddit.com/api/v1/authorize"
params = {
"client_id": self.client_id,
auth_params = {
"client_id": client_id,
"response_type": "code",
"state": self.state,
"state": state,
"redirect_uri": self.redirect_uri,
"duration": "permanent",
"scope": "read",
"scope": scope,
}
# receive 'code'
params = self.open(url, params)
params = self.open(auth_url, auth_params)
if self.state != params.get("state"):
# check auth response
if state != params.get("state"):
self.send("'state' mismatch: expected {}, got {}.".format(
self.state, params.get("state")))
state, params.get("state")))
return
if "error" in params:
self.send(params["error"])
return
# exchange 'code' for 'refresh_token'
url = "https://www.reddit.com/api/v1/access_token"
auth = (self.client_id, "")
data = {
"grant_type": "authorization_code",
"code": params["code"],
"redirect_uri": self.redirect_uri,
}
data = self.session.post(url, auth=auth, data=data).json()
auth = (client_id, client_secret)
data = self.session.post(token_url, data=data, auth=auth).json()
# check token response
if "error" in data:
self.send(data["error"])
else:
self.send(REDDIT_MSG_TEMPLATE.format(token=data["refresh_token"]))
return
# display refresh token
self.send(OAUTH2_MSG_TEMPLATE.format(
category=self.subcategory,
token=data["refresh_token"]
))
class OAuthReddit(OAuthBase):
subcategory = "reddit"
pattern = ["oauth:reddit$"]
def items(self):
yield Message.Version, 1
self.session.headers["User-Agent"] = reddit.RedditAPI.USER_AGENT
self._oauth2_authorization_code_grant(
reddit.RedditAPI.CLIENT_ID,
"",
"https://www.reddit.com/api/v1/authorize",
"https://www.reddit.com/api/v1/access_token",
"read",
)
class OAuthDeviantart(OAuthBase):
subcategory = "deviantart"
pattern = ["oauth:deviantart$"]
redirect_uri = "https://mikf.github.io/gallery-dl/oauth-redirect.html"
def items(self):
yield Message.Version, 1
self._oauth2_authorization_code_grant(
deviantart.DeviantartAPI.CLIENT_ID,
deviantart.DeviantartAPI.CLIENT_SECRET,
"https://www.deviantart.com/oauth2/authorize",
"https://www.deviantart.com/oauth2/token",
"browse",
)
class OAuthFlickr(OAuthBase):
@ -128,7 +159,7 @@ class OAuthFlickr(OAuthBase):
pattern = ["oauth:flickr$"]
def __init__(self, match):
OAuthBase.__init__(self)
OAuthBase.__init__(self, match)
self.session = util.OAuthSession(
self.session,
flickr.FlickrAPI.API_KEY, flickr.FlickrAPI.API_SECRET
@ -162,17 +193,18 @@ class OAuthFlickr(OAuthBase):
token_secret=data["oauth_token_secret"][0]))
REDDIT_MSG_TEMPLATE = """
OAUTH2_MSG_TEMPLATE = """
Your Refresh Token is
{token}
Put this value into your configuration file as 'extractor.reddit.refesh-token'.
Put this value into your configuration file as
'extractor.{category}.refesh-token'.
Example:
{{
"extractor": {{
"reddit": {{
"{category}": {{
"refresh-token": "{token}"
}}
}}

17
scripts/build_supportedsites.py
Unescape View File

@ -76,14 +76,15 @@ SUBCATEGORY_MAP = {
}
AUTH_MAP = {
"batoto" : "Optional",
"exhentai": "Optional",
"flickr" : "Optional (OAuth)",
"nijie" : "Required",
"pixiv" : "Required",
"reddit" : "Optional (OAuth)",
"sankaku" : "Optional",
"seiga" : "Required",
"batoto" : "Optional",
"deviantart": "Optional (OAuth)",
"exhentai" : "Optional",
"flickr" : "Optional (OAuth)",
"nijie" : "Required",
"pixiv" : "Required",
"reddit" : "Optional (OAuth)",
"sankaku" : "Optional",
"seiga" : "Required",
}
IGNORE_LIST = (

3
test/test_extractors.py
Unescape View File

@ -82,8 +82,7 @@ skip = [
"exhentai", "kissmanga", "mangafox", "dynastyscans", "nijie",
"archivedmoe", "archiveofsins", "thebarchive",
# temporary issues
"mangareader",
"mangapanda",
"hbrowse",
]
# enable selective testing for direct calls
if __name__ == '__main__' and len(sys.argv) > 1:

Write Preview
Loading…
Cancel
Save