Create a script to keep the sourcecode up-to-date

The update.py script will push changes to a git repo when changes are
found. See the README for more details.

.gitignore

@@ -0,0 +1,2 @@
+*.swp
+*.swo

README.md

@@ -8,3 +8,23 @@ Given the main JS file URL, the source code can be downloaded and mapped using:
 ```
 
 For convenience you can run `python update.py`
+
+## Syncing with git repo
+
+The script is designed to keep the contents of a git repo up-to-date with the
+current files. This can be paired with a cronjob to keep an up-to-date version
+of the code someplace.
+
+To ensure the `push` phase of the update works as expected, it is recommended
+to provide an SSH URL for the repository, as we can leverage an SSH key for a
+password-less authentication.
+
+The following example would keep the latest code extracted from dndbeyond.com
+stored in the `ddb_src` directory on the `master` branch of the given git repo.
+If changes are found they are automatically pushed to the repository,
+```
+python update.py \
+ --directory "ddb_src" \
+ --git-repo "ssh://git@git.example.com/test/dndbeyond_src.git"
+ --git-branch "master"
+```

update.py

@@ -3,8 +3,12 @@ import argparse
 import gzip
 import html.parser
 import logging
+import pathlib
 import re
+import shutil
 import subprocess
+import tempfile
+import typing
 import urllib.request
 
 ## The URL of a dndbeyond character which won't be deleted
@@ -21,6 +25,25 @@ def main():
     parser = argparse.ArgumentParser(
         description="Utility for scanning and converting videos."
     )
+    parser.add_argument(
+        "-d",
+        "--directory",
+        required=True,
+        help="The destination directory to put the files (required)",
+    )
+    parser.add_argument(
+        "-r",
+        "--git-repo",
+        type=str,
+        help="Store the code in the directory of this git repo",
+    )
+    parser.add_argument(
+        "-b",
+        "--git-branch",
+        type=str,
+        default="master",
+        help="Git branch to push to",
+    )
     parser.add_argument(
         "-u",
         "--character-url",
@@ -43,16 +66,11 @@ def main():
     char_data = download_character()
     parser = MainJSExtractor()
     parser.feed(char_data.decode("utf-8"))
-    print(f"Found URL: {parser.js_url}")
+    logging.debug(f"Found URL: {parser.js_url}")
-    subprocess.check_call(
+    if args.git_repo is None:
-        [
+        download_src(args.directory, parser.js_url)
-            SOURCEMAPPER_BIN,
+    else:
-            "-output",
+        update_repo(args.git_repo, args.git_branch, args.directory, parser.js_url)
-            "ddb_main",
-            "-jsurl",
-            parser.js_url,
-        ]
-    )
 
 
 def download_character() -> bytes:
@@ -95,5 +113,75 @@ class MainJSExtractor(html.parser.HTMLParser):
                         self.js_url = attr_val
 
 
+def download_src(output_dir: str, js_url: str) -> None:
+    subprocess.run(
+        [
+            SOURCEMAPPER_BIN,
+            "-output",
+            output_dir,
+            "-jsurl",
+            js_url,
+        ],
+        check=True,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+
+
+def update_repo(git_repo: str, git_branch: str, output_dir: str, js_url: str) -> None:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        logging.debug(f"Cloning repo {git_repo} to {tmpdir}")
+        git_sparse_clone(tmpdir, git_repo, git_branch, output_dir)
+        checkout_output_dir = pathlib.Path(tmpdir) / output_dir
+        logging.debug(f"Updating source code in {checkout_output_dir}")
+        if checkout_output_dir.exists():
+            shutil.rmtree(str(checkout_output_dir))
+        download_src(str(checkout_output_dir), js_url)
+        git_commit_all(tmpdir, "New source found from dndbeyond.com")
+
+
+def git_sparse_clone(
+    work_tree: str, git_repo: str, git_branch: str, sparse_dir: str
+) -> None:
+    # Clone without checkout first so we can do a sparse checkout
+    git_empty_checkout(work_tree, git_repo)
+    git_setup_sparse_checkout(work_tree, sparse_dir)
+    git_checkout(work_tree, git_branch)
+
+
+def git_empty_checkout(work_tree: str, git_repo: str) -> None:
+    run_git_cmd(["clone", "--no-checkout", git_repo, work_tree], None)
+
+
+def git_setup_sparse_checkout(work_tree: str, sparse_dir: str) -> None:
+    run_git_cmd(["sparse-checkout", "set", "--no-cone", sparse_dir], work_tree)
+
+
+def git_checkout(work_tree: str, git_branch: str) -> None:
+    run_git_cmd(["checkout", git_branch], work_tree)
+
+
+def git_commit_all(work_tree: str, commit_msg: str) -> None:
+    run_git_cmd(["add", "--all"], work_tree)
+    git_status = run_git_cmd(["status", "--porcelain=v1"], work_tree)
+    if len(git_status.stdout) > 0:
+        logging.debug(f"Changes found, comitting")
+        run_git_cmd(["commit", "-m", commit_msg], work_tree)
+        run_git_cmd(["push"], work_tree)
+    else:
+        logging.debug("No changes found")
+
+
+def run_git_cmd(args: typing.List[str], work_tree: str) -> subprocess.CompletedProcess:
+    git_cmd = ["/usr/bin/git"]
+    if work_tree is not None:
+        git_dir = f"{work_tree}/.git"
+        git_cmd += [
+            f"--git-dir={git_dir}",
+            f"--work-tree={work_tree}",
+        ]
+    return subprocess.run(git_cmd + args, check=True, capture_output=True)
+
+
 if __name__ == "__main__":
     main()