parent
ad73789256
commit
157bbf31fe
@ -0,0 +1,218 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
|
||||
import re
|
||||
|
||||
from .common import Extractor, Message
|
||||
from .. import text, exception
|
||||
from ..cache import cache
|
||||
|
||||
|
||||
class GirlswithmuscleExtractor(Extractor):
|
||||
def login(self):
|
||||
username, password = self._get_auth_info()
|
||||
if username:
|
||||
self.cookies_update(self._login_impl(username, password))
|
||||
|
||||
@staticmethod
|
||||
def _is_logged_in(page_text: str) -> bool:
|
||||
return 'Log in' not in page_text
|
||||
|
||||
@staticmethod
|
||||
def _get_csrfmiddlewaretoken(page: str) -> str:
|
||||
return text.extract(
|
||||
page,
|
||||
'name="csrfmiddlewaretoken" value="',
|
||||
'"'
|
||||
)[0]
|
||||
|
||||
def _open_login_page(self):
|
||||
"""We need it to get second CSRF token"""
|
||||
url = "https://www.girlswithmuscle.com/login/?next=/"
|
||||
response = self.request(url)
|
||||
return self._get_csrfmiddlewaretoken(response.text)
|
||||
|
||||
def _send_login_request(self, username, password, csrf_mw):
|
||||
"""Actual login action"""
|
||||
data = {
|
||||
"csrfmiddlewaretoken": csrf_mw,
|
||||
"username": username,
|
||||
"password": password,
|
||||
"next": "/"
|
||||
}
|
||||
|
||||
# Otherwise will be 403 Forbidden
|
||||
self.session.headers['Origin'] = 'https://www.girlswithmuscle.com'
|
||||
self.session.headers['Referer'] = \
|
||||
'https://www.girlswithmuscle.com/login/?next=/'
|
||||
|
||||
# if successful, will update cookies
|
||||
url = "https://www.girlswithmuscle.com/login/"
|
||||
response = self.request(url, method="post", data=data)
|
||||
|
||||
if "Wrong username or password" in response.text:
|
||||
raise exception.AuthenticationError()
|
||||
elif not self._is_logged_in(response.text):
|
||||
raise exception.AuthenticationError("Account data is missing")
|
||||
|
||||
@cache(maxage=28 * 86400, keyarg=1)
|
||||
def _login_impl(self, username, password):
|
||||
self.log.info("Logging in as %s", username)
|
||||
|
||||
csrf_mw = self._open_login_page()
|
||||
self._send_login_request(username, password, csrf_mw)
|
||||
return {c.name: c.value for c in self.session.cookies}
|
||||
|
||||
|
||||
class GirlswithmusclePostExtractor(GirlswithmuscleExtractor):
|
||||
"""Extractor for individual posts on girlswithmuscle.com"""
|
||||
category = "girlswithmuscle"
|
||||
subcategory = "post"
|
||||
directory_fmt = ("{category}", "{model}")
|
||||
filename_fmt = "{model}_{id}.{extension}"
|
||||
archive_fmt = "{type}_{model}_{id}"
|
||||
pattern = (r"(?:https?://)?(?:www\.)?girlswithmuscle\.com"
|
||||
r"/(\d+)/")
|
||||
example = "https://www.girlswithmuscle.com/1841638/"
|
||||
|
||||
def __init__(self, match):
|
||||
Extractor.__init__(self, match)
|
||||
self.id = match.groups()[0]
|
||||
|
||||
def items(self):
|
||||
self.login()
|
||||
url = "https://girlswithmuscle.com/{}/".format(self.id)
|
||||
page = self.request(url).text
|
||||
|
||||
if page is None:
|
||||
raise exception.NotFoundError("post")
|
||||
|
||||
url = text.extr(page, 'class="main-image" src="', '"')
|
||||
if url:
|
||||
metadata = self.metadata(page, url, 'picture')
|
||||
else:
|
||||
url = text.extr(page, '<source src="', '"')
|
||||
metadata = self.metadata(page, url, 'video')
|
||||
|
||||
yield Message.Directory, metadata
|
||||
yield Message.Url, url, metadata
|
||||
|
||||
def metadata(self, page, url, content_type):
|
||||
info_source_begin = \
|
||||
'<div class="image-info" id="info-source" style="display: none">'
|
||||
info_source_end = "</div>"
|
||||
source = text.remove_html(
|
||||
text.extr(page, info_source_begin, info_source_end))
|
||||
|
||||
info_uploader_begin = '<div class="image-info" id="info-uploader">'
|
||||
info_uploader_end = "</div>"
|
||||
uploader = text.remove_html(
|
||||
text.extr(page, info_uploader_begin, info_uploader_end))
|
||||
|
||||
tags = text.extr(
|
||||
page, 'class="selected-tags">', "</span>", ''
|
||||
).split(', ')
|
||||
tags = [tag for tag in tags if tag]
|
||||
|
||||
score = text.parse_int(text.remove_html(
|
||||
text.extr(page, 'Score: <b>', '</span', '0')))
|
||||
|
||||
model = self._parse_model(page)
|
||||
|
||||
return {
|
||||
'id': self.id,
|
||||
'model': model,
|
||||
'model_list': self._parse_model_list(model),
|
||||
'tags': tags,
|
||||
'posted_dt': text.extr(
|
||||
page, 'class="hover-time" title="', '"', ''
|
||||
),
|
||||
'is_favorite': self._parse_is_favorite(page),
|
||||
'source_filename': source,
|
||||
'uploader': uploader,
|
||||
'score': score,
|
||||
'comments': self._parse_comments(page),
|
||||
'extension': text.ext_from_url(url),
|
||||
'type': content_type,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _parse_model(page):
|
||||
model = text.extr(page, '<title>', "</title>", None)
|
||||
return 'unknown' if model.startswith('Picture #') else model
|
||||
|
||||
@staticmethod
|
||||
def _parse_model_list(model):
|
||||
if model == 'unknown':
|
||||
return []
|
||||
else:
|
||||
return [name.strip() for name in model.split(',')]
|
||||
|
||||
@staticmethod
|
||||
def _parse_is_favorite(page):
|
||||
fav_button = text.extr(page, 'id="favorite-button">', "</span>", '')
|
||||
unfav_button = text.extr(page,
|
||||
'class="actionbutton unfavorite-button">',
|
||||
"</span>", '')
|
||||
|
||||
is_favorite = None
|
||||
if unfav_button == 'Unfavorite':
|
||||
is_favorite = True
|
||||
if fav_button == 'Favorite':
|
||||
is_favorite = False
|
||||
|
||||
return is_favorite
|
||||
|
||||
@staticmethod
|
||||
def _parse_comments(page):
|
||||
comments = text.extract_iter(page, '<div class="comment-body-inner">',
|
||||
'</div>')
|
||||
return [comment.strip() for comment in comments]
|
||||
|
||||
|
||||
class GirlswithmuscleGalleryExtractor(GirlswithmuscleExtractor):
|
||||
"""Extractor for galleries on girlswithmuscle.com"""
|
||||
category = "girlswithmuscle"
|
||||
subcategory = "gallery"
|
||||
pattern = r"(?:https?://)?(?:www\.)?girlswithmuscle\.com/images/(.*)"
|
||||
example = "https://www.girlswithmuscle.com/images/?name=Samantha%20Jerring"
|
||||
|
||||
def __init__(self, match):
|
||||
Extractor.__init__(self, match)
|
||||
self.query = match.groups()[0]
|
||||
|
||||
def pages(self):
|
||||
url = "https://www.girlswithmuscle.com/images/{}".format(self.query)
|
||||
response = self.request(url)
|
||||
if url != response.url:
|
||||
msg = ('Request was redirected to "{}", try logging in'.
|
||||
format(response.url))
|
||||
raise exception.AuthorizationError(msg)
|
||||
page = response.text
|
||||
|
||||
match = re.search(r"Page (\d+) of (\d+)", page)
|
||||
current, total = match.groups()
|
||||
current, total = text.parse_int(current), text.parse_int(total)
|
||||
|
||||
yield page
|
||||
for i in range(current + 1, total + 1):
|
||||
url = ("https://www.girlswithmuscle.com/images/{}/{}".
|
||||
format(i, self.query))
|
||||
yield self.request(url).text
|
||||
|
||||
def items(self):
|
||||
self.login()
|
||||
for page in self.pages():
|
||||
for imgid in text.extract_iter(page, 'id="imgid-', '"'):
|
||||
url = "https://www.girlswithmuscle.com/{}/".format(imgid)
|
||||
yield Message.Queue, url, {
|
||||
"gallery_name": self._parse_gallery_name(page),
|
||||
"_extractor": GirlswithmusclePostExtractor
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _parse_gallery_name(page):
|
||||
return text.extr(page, "<title>", "</title>")
|
@ -0,0 +1,56 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
|
||||
gallery_dl = __import__("gallery_dl.extractor.girlswithmuscle")
|
||||
_gwm = getattr(gallery_dl.extractor, "girlswithmuscle")
|
||||
|
||||
|
||||
__tests__ = (
|
||||
{
|
||||
"#url" : "https://www.girlswithmuscle.com/2136096/",
|
||||
"#category" : ("", "girlswithmuscle", "post"),
|
||||
"#class" : _gwm.GirlswithmusclePostExtractor,
|
||||
|
||||
'id' : '2136096',
|
||||
'model' : str,
|
||||
'tags' : list,
|
||||
'posted_dt' : '2023-12-12 16:04:03.438979+00:00',
|
||||
'source_filename': 'IMG_8714.png',
|
||||
'uploader' : 'toni1991',
|
||||
'score' : int,
|
||||
'extension' : 'png',
|
||||
"type" : 'picture',
|
||||
# These are not available, unless you're logged in
|
||||
'is_favorite' : None,
|
||||
'comments' : list,
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://www.girlswithmuscle.com/1841638/",
|
||||
"#category" : ("", "girlswithmuscle", "post"),
|
||||
"#class" : _gwm.GirlswithmusclePostExtractor,
|
||||
|
||||
'id' : '1841638',
|
||||
'model' : str,
|
||||
'tags' : list,
|
||||
'posted_dt' : '2022-08-16 17:20:16.006855+00:00',
|
||||
'source_filename': 'Snapinsta_299658611_1185267375661829_6167677658282784059_n.mp4',
|
||||
'uploader' : 'BriedFrain',
|
||||
'score' : int,
|
||||
'extension' : 'mp4',
|
||||
"type" : 'video',
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://www.girlswithmuscle.com/images/?name=Samantha%20Jerring",
|
||||
"#category" : ("", "girlswithmuscle", "gallery"),
|
||||
"#class" : _gwm.GirlswithmuscleGalleryExtractor,
|
||||
|
||||
"#count" : range(300, 3000),
|
||||
"gallery_name" : str
|
||||
},
|
||||
|
||||
)
|
Loading…
Reference in new issue