rclone/bin/make_manual.py

192 lines
5.1 KiB
Python
Raw Normal View History

2019-09-06 21:08:48 +00:00
#!/usr/bin/env python3
"""
Make single page versions of the documentation for release and
conversion into man pages etc.
"""
import os
import re
import time
from datetime import datetime
docpath = "docs/content"
outfile = "MANUAL.md"
# Order to add docs segments to make outfile
docs = [
"_index.md",
"install.md",
"docs.md",
"remote_setup.md",
"filtering.md",
2019-08-25 19:32:41 +00:00
"gui.md",
"rc.md",
"overview.md",
"flags.md",
2021-07-20 18:45:41 +00:00
"docker.md",
# Keep these alphabetical by full name
"fichier.md",
"alias.md",
"amazonclouddrive.md",
"s3.md",
"b2.md",
"box.md",
2017-11-12 17:54:25 +00:00
"cache.md",
2019-08-30 14:18:40 +00:00
"chunker.md",
"sharefile.md",
"crypt.md",
2020-12-02 00:47:07 +00:00
"compress.md",
"dropbox.md",
"filefabric.md",
"ftp.md",
"googlecloudstorage.md",
"drive.md",
"googlephotos.md",
2021-11-01 15:42:05 +00:00
"hasher.md",
"hdfs.md",
"http.md",
"hubic.md",
2018-08-07 18:51:12 +00:00
"jottacloud.md",
"koofr.md",
2019-09-09 20:56:16 +00:00
"mailru.md",
"mega.md",
"memory.md",
"netstorage.md",
"azureblob.md",
"onedrive.md",
"opendrive.md",
"qingstor.md",
2021-11-01 15:42:05 +00:00
"sia.md",
"swift.md",
2017-09-19 15:09:43 +00:00
"pcloud.md",
2019-08-26 14:25:20 +00:00
"premiumizeme.md",
2019-08-06 12:47:52 +00:00
"putio.md",
"seafile.md",
"sftp.md",
"storj.md",
2019-10-24 11:35:50 +00:00
"sugarsync.md",
"uptobox.md",
2018-09-07 14:08:29 +00:00
"union.md",
"webdav.md",
"yandex.md",
"zoho.md",
"local.md",
"changelog.md",
"bugs.md",
2015-07-30 19:26:29 +00:00
"faq.md",
"licence.md",
"authors.md",
"contact.md",
]
# Order to put the commands in - any not on here will be in sorted order
commands_order = [
"rclone_config.md",
"rclone_copy.md",
"rclone_sync.md",
"rclone_move.md",
"rclone_delete.md",
"rclone_purge.md",
"rclone_mkdir.md",
"rclone_rmdir.md",
"rclone_check.md",
"rclone_ls.md",
"rclone_lsd.md",
"rclone_lsl.md",
"rclone_md5sum.md",
"rclone_sha1sum.md",
"rclone_size.md",
"rclone_version.md",
"rclone_cleanup.md",
"rclone_dedupe.md",
]
# Docs which aren't made into outfile
ignore_docs = [
"downloads.md",
2015-08-19 21:10:04 +00:00
"privacy.md",
"donate.md",
]
def read_doc(doc):
"""Read file as a string"""
path = os.path.join(docpath, doc)
with open(path) as fd:
contents = fd.read()
parts = contents.split("---\n", 2)
if len(parts) != 3:
raise ValueError("Couldn't find --- markers: found %d parts" % len(parts))
contents = parts[2].strip()+"\n\n"
# Remove icons
contents = re.sub(r'<i class="fa.*?</i>\s*', "", contents)
# Interpret img shortcodes
# {{< img ... >}}
contents = re.sub(r'\{\{<\s*img\s+(.*?)>\}\}', r"<img \1>", contents)
# Make any img tags absolute
contents = re.sub(r'(<img.*?src=")/', r"\1https://rclone.org/", contents)
# Make [...](/links/) absolute
contents = re.sub(r'\]\((\/.*?\/(#.*)?)\)', r"](https://rclone.org\1)", contents)
# Add additional links on the front page
contents = re.sub(r'\{\{< rem MAINPAGELINK >\}\}', "- [Donate.](https://rclone.org/donate/)", contents)
2017-09-25 16:55:19 +00:00
# Interpret provider shortcode
# {{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}
contents = re.sub(r'\{\{<\s*provider.*?name="(.*?)".*?>\}\}', r"- \1", contents)
# Remove remaining shortcodes
contents = re.sub(r'\{\{<.*?>\}\}', r"", contents)
contents = re.sub(r'\{\{%.*?%\}\}', r"", contents)
return contents
def check_docs(docpath):
"""Check all the docs are in docpath"""
files = set(f for f in os.listdir(docpath) if f.endswith(".md"))
files -= set(ignore_docs)
docs_set = set(docs)
if files == docs_set:
return
2019-09-06 21:08:48 +00:00
print("Files on disk but not in docs variable: %s" % ", ".join(files - docs_set))
print("Files in docs variable but not on disk: %s" % ", ".join(docs_set - files))
raise ValueError("Missing files")
def read_command(command):
doc = read_doc("commands/"+command)
doc = re.sub(r"### Options inherited from parent commands.*$", "", doc, 0, re.S)
doc = doc.strip()+"\n"
return doc
def read_commands(docpath):
"""Reads the commands an makes them into a single page"""
files = set(f for f in os.listdir(docpath + "/commands") if f.endswith(".md"))
docs = []
for command in commands_order:
docs.append(read_command(command))
files.remove(command)
for command in sorted(files):
if command != "rclone.md":
docs.append(read_command(command))
return "\n".join(docs)
def main():
check_docs(docpath)
2019-09-06 21:08:48 +00:00
command_docs = read_commands(docpath).replace("\\", "\\\\") # escape \ so we can use command_docs in re.sub
build_date = datetime.utcfromtimestamp(
int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))
with open(outfile, "w") as out:
out.write("""\
%% rclone(1) User Manual
%% Nick Craig-Wood
%% %s
""" % build_date.strftime("%b %d, %Y"))
for doc in docs:
contents = read_doc(doc)
# Substitute the commands into doc.md
if doc == "docs.md":
contents = re.sub(r"The main rclone commands.*?for the full list.", command_docs, contents, 0, re.S)
out.write(contents)
2019-09-06 21:08:48 +00:00
print("Written '%s'" % outfile)
if __name__ == "__main__":
main()