implement 'parent-directory' option (#551)

docs/configuration.rst

@@ -108,6 +108,17 @@ Description Directory path used as the base for all download destinations.
 =========== =====
 
 
+extractor.*.parent-directory
+----------------------------
+=========== =====
+Type        ``bool``
+Default     ``false``
+Description Use an extractor's current target directory as
+            `base-directory <extractor.*.base-directory_>`__
+            for any spawned child extractors.
+=========== =====
+
+
 extractor.*.path-restrict
 -------------------------
 =========== =====

gallery_dl/extractor/common.py

@@ -39,6 +39,7 @@ class Extractor():
 
         self._cookiefile = None
         self._cookiejar = self.session.cookies
+        self._parentdir = ""
         self._retries = self.config("retries", 4)
         self._timeout = self.config("timeout", 30)
         self._verify = self.config("verify", True)

gallery_dl/extractor/reddit.py

@@ -31,6 +31,7 @@ class RedditExtractor(Extractor):
         match_subreddit = RedditSubredditExtractor.pattern.match
         match_user = RedditUserExtractor.pattern.match
 
+        parentdir = self.config("parent-directory")
         submissions = self.submissions()
         visited = set()
         depth = 0
@@ -56,6 +57,8 @@ class RedditExtractor(Extractor):
                         yield Message.Url, "ytdl:" + url, submission
                     elif not submission["is_self"]:
                         urls.append((url, submission))
+                elif parentdir:
+                    yield Message.Directory, comments[0]
 
                 if self.api.comments:
                     if submission:

gallery_dl/job.py

@@ -182,7 +182,14 @@ class DownloadJob(Job):
         self.downloaders = {}
         self.postprocessors = None
         self.out = output.select()
-        self.visited = parent.visited if parent else set()
+
+        if parent:
+            self.visited = parent.visited
+            pfmt = parent.pathfmt
+            if pfmt and parent.extractor.config("parent-directory"):
+                self.extractor._parentdir = pfmt.directory
+        else:
+            self.visited = set()
 
     def handle_url(self, url, kwdict, fallback=None):
         """Download the resource specified in 'url'"""

gallery_dl/util.py

@@ -627,12 +627,14 @@ class PathFormat():
         self.delete = False
         self.path = self.realpath = self.temppath = ""
 
-        basedir = expand_path(
-            extractor.config("base-directory", (".", "gallery-dl")))
-        if os.altsep and os.altsep in basedir:
-            basedir = basedir.replace(os.altsep, os.sep)
-        if basedir[-1] != os.sep:
-            basedir += os.sep
+        basedir = extractor._parentdir
+        if not basedir:
+            basedir = expand_path(
+                extractor.config("base-directory", (".", "gallery-dl")))
+            if os.altsep and os.altsep in basedir:
+                basedir = basedir.replace(os.altsep, os.sep)
+            if basedir[-1] != os.sep:
+                basedir += os.sep
         self.basedirectory = basedir
 
         restrict = extractor.config("path-restrict", "auto")