From 53070de86d4c0c91ef7e4edb90709ee9769c343a Mon Sep 17 00:00:00 2001 From: Matthew Date: Wed, 3 Oct 2018 14:37:31 -0700 Subject: [PATCH] Add subtitle field and allow interactive specification of headers. --- knowledge_repo/_version.py | 1 + ...7e1_add_subtitle_field_to_post_database.py | 22 +++ knowledge_repo/app/models.py | 2 + knowledge_repo/app/static/css/pages/base.css | 13 +- .../app/static/css/pages/markdown-base.css | 26 +++- .../app/templates/markdown-rendered.html | 3 +- knowledge_repo/app/utils/render.py | 22 +-- knowledge_repo/converter.py | 14 +- knowledge_repo/converters/ipynb.py | 2 +- knowledge_repo/converters/md.py | 2 +- knowledge_repo/converters/pkp.py | 2 + knowledge_repo/converters/rmd.py | 2 +- knowledge_repo/post.py | 136 ++++++++++++++---- .../postprocessors/format_checks.py | 21 +-- 14 files changed, 195 insertions(+), 73 deletions(-) create mode 100644 knowledge_repo/app/migrations/versions/d15f6cac07e1_add_subtitle_field_to_post_database.py diff --git a/knowledge_repo/_version.py b/knowledge_repo/_version.py index 221aa144d..b8403f5d8 100644 --- a/knowledge_repo/_version.py +++ b/knowledge_repo/_version.py @@ -26,6 +26,7 @@ 'gitpython', # Git abstraction 'tabulate', # Rendering information prettily in knowledge_repo script 'pyyaml', # Used to configure knowledge repositories + 'cooked_input', # Used for interactive input from user in CLI tooling # Flask App Dependencies 'flask', # Main flask framework diff --git a/knowledge_repo/app/migrations/versions/d15f6cac07e1_add_subtitle_field_to_post_database.py b/knowledge_repo/app/migrations/versions/d15f6cac07e1_add_subtitle_field_to_post_database.py new file mode 100644 index 000000000..61530434f --- /dev/null +++ b/knowledge_repo/app/migrations/versions/d15f6cac07e1_add_subtitle_field_to_post_database.py @@ -0,0 +1,22 @@ +"""Add subtitle field to post database. + +Revision ID: d15f6cac07e1 +Revises: 009eafe4838f +Create Date: 2018-10-03 12:31:18.462880 + +""" + +# revision identifiers, used by Alembic. +revision = 'd15f6cac07e1' +down_revision = '009eafe4838f' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('posts', sa.Column('subtitle', sa.Text(), nullable=True)) + + +def downgrade(): + op.drop_column('posts', 'subtitle') diff --git a/knowledge_repo/app/models.py b/knowledge_repo/app/models.py index 8b6d71dea..92c1f5c3a 100644 --- a/knowledge_repo/app/models.py +++ b/knowledge_repo/app/models.py @@ -365,6 +365,7 @@ class Post(db.Model): revision = db.Column(db.Integer()) title = db.Column(db.Text()) + subtitle = db.Column(db.Text()) tldr = db.Column(db.Text) keywords = db.Column(db.Text) thumbnail = db.Column(db.Text()) @@ -597,6 +598,7 @@ def update_metadata_from_kp(self, kp): self.repository = kp.repository_uri self.revision = kp.revision self.title = headers['title'] + self.subtitle = headers.get('subtitle') self.tldr = headers['tldr'] self.authors = headers.get('authors', []) self.tags = headers.get('tags', []) diff --git a/knowledge_repo/app/static/css/pages/base.css b/knowledge_repo/app/static/css/pages/base.css index 23cfd59f8..f6207beef 100644 --- a/knowledge_repo/app/static/css/pages/base.css +++ b/knowledge_repo/app/static/css/pages/base.css @@ -1,5 +1,5 @@ .container-fluid { - max-width: 110em; + max-width: 100em; margin: auto; } @@ -307,7 +307,7 @@ body { color: #565a5c; background-color: whitesmoke; - font-size: 14px; + font-size: 12pt; font-family: 'Lato', sans-serif; /*background-color: #F8F8F8;*/ } @@ -328,11 +328,12 @@ ul { margin-bottom: 10px; } -h1 { font-size: 2em; margin: .67em 0 } +h1 { font-size: 1.8em; margin: .67em 0 } h2 { font-size: 1.5em; margin: .75em 0 } -h3 { font-size: 1.17em; margin: .83em 0 } -h5 { font-size: .83em; margin: 1.5em 0 } -h6 { font-size: .75em; margin: 1.67em 0 } +h3 { font-size: 1.3em; margin: .83em 0 } +h4 { font-size: 1.1em; margin: 1em 0 } +h5 { font-size: 1em; margin: 1.5em 0 } +h6 { font-size: 1em; margin: 1.67em 0; font-style: italic;} h1, h2, h3, h4, h5, h6 { font-weight: bolder } diff --git a/knowledge_repo/app/static/css/pages/markdown-base.css b/knowledge_repo/app/static/css/pages/markdown-base.css index 542516901..f48f8f965 100644 --- a/knowledge_repo/app/static/css/pages/markdown-base.css +++ b/knowledge_repo/app/static/css/pages/markdown-base.css @@ -7,6 +7,7 @@ margin-bottom: 2em; position: relative; box-shadow: 0px 2px 3px rgba(0,0,0,0.1); + font-size: 12pt; } @media (max-width: 992px) { @@ -19,24 +20,37 @@ } } +.renderedMarkdown span.title, .renderedMarkdown h1, .renderedMarkdown h2, .renderedMarkdown h3, .renderedMarkdown h4, .renderedMarkdown h5, .renderedMarkdown h6 { + color: black; margin-top: 1.5em; margin-bottom: 0.5em; font-weight: bold; } -.renderedMarkdown h1:nth-of-type(1) { +.renderedMarkdown span.title:nth-of-type(1) { margin-top: 1em; text-align: center; display: block; + font-size: 2.2em; + margin-top: .67em; + margin-bottom: .3em; } -.renderedMarkdown h2 { +.renderedMarkdown span.subtitle { + text-align: center; + display: block; + font-size: 1.5em; + font-style: italic; + margin-bottom: .3em; +} + +.renderedMarkdown h1 { border-bottom: 1px solid #eee; padding-bottom: 0.3em; } @@ -171,8 +185,8 @@ .renderedMarkdown .codehilite + .code-output { margin-top: -1em; - margin-left: -21px; - margin-right: -21px; + margin-left: -1.5em; + margin-right: -1.5em; font-family: monospace; } @@ -230,8 +244,8 @@ .renderedMarkdown .codehilite { margin: 1em; - margin-left: -21px; - margin-right: -21px; + margin-left: -1.5em; + margin-right: -1.5em; padding: 0px; } diff --git a/knowledge_repo/app/templates/markdown-rendered.html b/knowledge_repo/app/templates/markdown-rendered.html index 929d6ccb9..f6737ac2a 100644 --- a/knowledge_repo/app/templates/markdown-rendered.html +++ b/knowledge_repo/app/templates/markdown-rendered.html @@ -202,8 +202,7 @@

{{ comments | length }} Comments

}) //Turn all the headers to be links -//h1 = Title, don't want that -var all_headers = [$("h2"), $("h3"), $("h4"), $("h5"), $("h6")] +var all_headers = [$("h1"), $("h2"), $("h3"), $("h4"), $("h5"), $("h6")] $.each(all_headers, function(index, value){ $.each(value, function(i, v){ var inner_html = v.innerHTML diff --git a/knowledge_repo/app/utils/render.py b/knowledge_repo/app/utils/render.py index 18079eeed..c7042fa40 100644 --- a/knowledge_repo/app/utils/render.py +++ b/knowledge_repo/app/utils/render.py @@ -5,6 +5,7 @@ import pygments from flask import url_for +from jinja2 import Template from knowledge_repo.post import KnowledgePost MARKDOWN_EXTENSIONS = ['extra', @@ -33,29 +34,32 @@ def render_post_tldr(post): def render_post_header(post): - header_template = u""" + header_template = Template(u"""
-

{title}

- {authors} - {date_created} - (Last Updated: {date_updated}) - {tldr} + {{title}} + {% if subtitle %}{{subtitle}}{% endif %} + {{authors}} + {{date_created}} + (Last Updated: {{date_updated}}) + {{tldr}}
- """ + """) def get_authors(usernames, authors): authors = [u"{}".format(url_for('index.render_feed', authors=username), author) for username, author in zip(usernames, authors)] return u' and '.join(u', '.join(authors).rsplit(', ', 1)) if isinstance(post, KnowledgePost): - return header_template.format(title=post.headers['title'], + return header_template.render(title=post.headers['title'], + subtitle=post.headers.get('subtitle'), authors=get_authors(post.headers['authors'], post.headers['authors']), date_created=post.headers['created_at'].strftime("%B %d, %Y"), date_updated=post.headers['updated_at'].strftime("%B %d, %Y"), tldr=render_post_tldr(post)) else: - return header_template.format(title=post.title, + return header_template.render(title=post.title, + subtitle=post.subtitle, authors=get_authors([author.identifier for author in post.authors], [author.format_name for author in post.authors]), date_created=post.created_at.strftime("%B %d, %Y"), date_updated=post.updated_at.strftime("%B %d, %Y"), diff --git a/knowledge_repo/converter.py b/knowledge_repo/converter.py index c9eabbaa8..f4f6ce574 100644 --- a/knowledge_repo/converter.py +++ b/knowledge_repo/converter.py @@ -20,13 +20,14 @@ def get_format(filename, format=None): class KnowledgePostConverter(with_metaclass(SubclassRegisteringABCMeta, object)): _registry_keys = None # File extensions - def __init__(self, kp, format=None, postprocessors=None, **kwargs): + def __init__(self, kp, format=None, postprocessors=None, interactive=False, **kwargs): check_dependencies(self.dependencies, "Whoops! You are missing some dependencies required to use `{}` instances.".format(self.__class__.__name__)) self.kp = kp self.format = format if postprocessors is None: postprocessors = [('extract_images', {}), ('format_checks', {})] self.postprocessors = postprocessors + self.interactive = interactive self.init(**kwargs) @property @@ -60,6 +61,9 @@ def __getattribute__(self, attr): return self.__get_wrapped_with_postprocessors(object.__getattribute__(self, attr)) return object.__getattribute__(self, attr) + def kp_write(self, md, headers=None, images={}): + return self.kp.write(md, headers=headers, images=images, interactive=self.interactive) + def from_file(self, filename, **opts): raise NotImplementedError @@ -73,12 +77,12 @@ def to_string(self, **opts): raise NotImplementedError @classmethod - def for_file(cls, kp, filename, format=None, postprocessors=None): - return cls.for_format(kp, get_format(filename, format), postprocessors=postprocessors) + def for_file(cls, kp, filename, format=None, postprocessors=None, interactive=False): + return cls.for_format(kp, get_format(filename, format), postprocessors=postprocessors, interactive=interactive) @classmethod - def for_format(cls, kp, format, postprocessors=None): + def for_format(cls, kp, format, postprocessors=None, interactive=False): if format.lower() not in cls._registry: raise ValueError("The knowledge repository does not support files of type '{}'. Supported types are: {}." .format(format, ','.join(list(cls._registry.keys())))) - return cls._get_subclass_for(format.lower())(kp, format=format, postprocessors=postprocessors) + return cls._get_subclass_for(format.lower())(kp, format=format, postprocessors=postprocessors, interactive=interactive) diff --git a/knowledge_repo/converters/ipynb.py b/knowledge_repo/converters/ipynb.py index d25c87c5e..c0e03a193 100644 --- a/knowledge_repo/converters/ipynb.py +++ b/knowledge_repo/converters/ipynb.py @@ -68,7 +68,7 @@ def from_file(self, filename): dl], template_file='full.tpl') (body, resources) = md_exporter.from_notebook_node(nb) - self.kp.write(body, images={name.split( + self.kp_write(body, images={name.split( 'images/')[1]: data for name, data in resources.get('outputs', {}).items()}) # Add cleaned ipynb file diff --git a/knowledge_repo/converters/md.py b/knowledge_repo/converters/md.py index a5753867d..259ef3348 100644 --- a/knowledge_repo/converters/md.py +++ b/knowledge_repo/converters/md.py @@ -6,4 +6,4 @@ class MdConverter(KnowledgePostConverter): def from_file(self, filename): with open(filename) as f: - self.kp.write(f.read()) + self.kp_write(f.read()) diff --git a/knowledge_repo/converters/pkp.py b/knowledge_repo/converters/pkp.py index 01a7b4767..84b37e85b 100644 --- a/knowledge_repo/converters/pkp.py +++ b/knowledge_repo/converters/pkp.py @@ -27,6 +27,8 @@ def to_string(self): return data.read() def from_file(self, filename): + # Note: header checks are not applied here, since it should not be + # possible to create portable knowledge post with incorrect headers. zf = zipfile.ZipFile(filename, 'r') for ref in zf.namelist(): diff --git a/knowledge_repo/converters/rmd.py b/knowledge_repo/converters/rmd.py index 921a4f6fd..f41f7f334 100644 --- a/knowledge_repo/converters/rmd.py +++ b/knowledge_repo/converters/rmd.py @@ -39,7 +39,7 @@ def from_file(self, filename, rebuild=True): Rmd_filename = tmp_path with open(Rmd_filename) as f: - self.kp.write(f.read()) + self.kp_write(f.read()) self.kp.add_srcfile(filename) # Clean up temporary file diff --git a/knowledge_repo/post.py b/knowledge_repo/post.py index 02c7978c5..a26abb4e5 100755 --- a/knowledge_repo/post.py +++ b/knowledge_repo/post.py @@ -1,4 +1,8 @@ from __future__ import absolute_import +from builtins import next +from builtins import object +from collections import namedtuple +import itertools import os import posixpath import re @@ -10,15 +14,43 @@ import base64 import uuid +import cooked_input as ci import six -from builtins import next, object from .utils.encoding import encode, decode logger = logging.getLogger(__name__) +# Define available headers, their types, and runtime input specification +Header = namedtuple('Header', ('name', 'type', 'input')) -SAMPLE_HEADER = """ +HEADER_REQUIRED_FIELD_TYPES = [ + Header('title', six.string_types, ci.GetInput(prompt='title')), + Header('authors', list, ci.GetInput(prompt='authors (comma separated)', convertor=ci.ListConvertor())), + Header('tldr', six.string_types, ci.GetInput(prompt='tldr')), + Header('created_at', datetime.datetime, ci.GetInput(prompt='created_at', convertor=ci.DateConvertor(), default=datetime.date.today())), +] + +HEADER_OPTIONAL_FIELD_TYPES = [ + Header('subtitle', six.string_types, ci.GetInput(prompt='subtitle', required=False)), + Header('tags', list, ci.GetInput(prompt='tags (comma separated)', convertor=ci.ListConvertor(), required=False)), + Header('path', six.string_types, ci.GetInput(prompt='path', required=False)), + Header('updated_at', datetime.datetime, ci.GetInput(prompt='updated_at', convertor=ci.DateConvertor(), default=datetime.datetime.now())), + Header('private', bool, ci.GetInput(prompt='private', convertor=ci.BooleanConvertor(), required=False)), # If true, this post starts out private + Header('allowed_groups', list, ci.GetInput(prompt='allowed_groups (comma separated)', convertor=ci.ListConvertor(), required=False)), + Header('thumbnail', (int, ) + six.string_types, ci.GetInput(prompt='thumbnail', required=False)), +] + +HEADERS_ALL = { + header.name: header + for header in itertools.chain(HEADER_REQUIRED_FIELD_TYPES, HEADER_OPTIONAL_FIELD_TYPES) +} + +# Headers to prompt for if missing when in interactive mode +HEADERS_INTERACTIVE = ['title', 'subtitle', 'authors', 'tldr', 'created_at', 'tags'] + + +HEADER_SAMPLE = """ --- title: "This is a Knowledge Post title, quoted so we can use special characters like ':'" authors: @@ -216,7 +248,7 @@ def read(self, images=False, headers=True, body=True): md = decode(self._read_ref('knowledge.md')) mtch = re.match('^---\n[\\s\\S]+?---\n', md) if not mtch: - raise ValueError("YAML header is missing. Please ensure that the top of your post has a header of the following form:\n" + SAMPLE_HEADER) + raise ValueError("YAML header is missing. Please ensure that the top of your post has a header of the following form:\n" + HEADER_SAMPLE) if not headers: md = re.sub('^---\n[\\s\\S]+?---\n', '', md, count=1) if not body: @@ -254,13 +286,22 @@ def read_src(self, ref): return self._read_ref('orig_src/' + ref) raise e - def write(self, md, headers=None, images={}): + def write(self, md, headers=None, images={}, interactive=False): md = md.strip() - if headers is not None: - md = re.sub('^---\n[\\s\\S]+?---\n', '', md, count=1) - md = '---\n' + \ - yaml.safe_dump(headers, default_flow_style=False) + '---\n' + md - md += '\n' + + if not headers: + headers = self._get_headers_from_yaml(md) + + md = re.sub(r'^---\n[\s\S]+?---\n', '', md, count=1) + + headers = self._verify_headers(headers, interactive=interactive) + + md = ( # Format with unicode seems to have issue in Python 2, so we explicitly concatenate + '---\n' + + yaml.safe_dump(headers, default_flow_style=False) + + '---\n\n' + + md + ) self._write_ref('knowledge.md', encode(md)) @@ -285,15 +326,62 @@ def add_srcfile(self, filename, name=None): # ------------- Knowledge Post Format ---------------------------------- - @property - def headers(self): + def _get_headers_from_yaml(self, yaml_str): try: - headers = next(yaml.load_all(self.read(body=False))) - except StopIteration as e: - raise ValueError("YAML header is missing. Please ensure that the top of your post has a header of the following form:\n" + SAMPLE_HEADER) + return next(yaml.load_all(yaml_str)) except yaml.YAMLError as e: - raise ValueError( - "YAML header is incorrectly formatted or missing. The following information may be useful:\n{}\nIf you continue to have difficulties, try pasting your YAML header into an online parser such as http://yaml-online-parser.appspot.com/.".format(str(e))) + logger.info( + "YAML header is incorrectly formatted or missing. The following " + "information may be useful:\n{}\nIf you continue to have " + "difficulties, try pasting your YAML header into an online parser " + "such as http://yaml-online-parser.appspot.com/.".format(str(e)) + ) + except StopIteration as e: + logger.info('YAML header is missing!') + return {} + + def _verify_headers(self, headers, interactive=False): + missing_required_headers = ( + set(h.name for h in HEADER_REQUIRED_FIELD_TYPES).difference(headers) + ) + + if interactive: + missing_suggested_headers = ( + set(HEADERS_INTERACTIVE).difference(headers).difference(missing_required_headers) + ) + + if missing_required_headers: + print("This post is missing the following required headers: {}".format(missing_required_headers)) + if missing_suggested_headers: + print("This post is missing the following suggested headers: {}".format(missing_suggested_headers)) + if missing_required_headers or missing_suggested_headers: + print( + "You will now be prompted for each missing header. If you wish " + "to abort the knowledge post creation, press Ctrl+C." + .format(missing_required_headers) + ) + + for header in HEADERS_INTERACTIVE: + if header not in headers: + headers[header] = HEADERS_ALL[header].input.get_input() + elif missing_required_headers: + raise RuntimeError( + "Knowledge post is missing required headers {}. Please rerun this " + "operation in interactive mode, or add headers manually to the " + "post source file.".format(missing_required_headers) + ) + + if 'tags' not in headers or not headers['tags']: + headers['tags'] = [] + headers['updated_at'] = datetime.datetime.now() + + return headers + + @property + def headers(self): + headers = self._get_headers_from_yaml(self.read(body=False)) + if not headers: + raise ValueError("YAML header is missing. Please ensure that the top of your post has a header of the following form:\n" + HEADER_SAMPLE) for key, value in headers.copy().items(): if isinstance(value, datetime.date): headers[key] = datetime.datetime.combine(value, datetime.time(0)) @@ -362,26 +450,26 @@ def web_uri(self): # Conversion/Import/Export methods @classmethod - def from_file(cls, filename, src_paths=[], format=None, postprocessors=None, **opts): - kp = KnowledgePostConverter.for_file(cls(), filename, format=format, postprocessors=postprocessors).from_file(filename, **opts) + def from_file(cls, filename, src_paths=[], format=None, postprocessors=None, interactive=False, **opts): + kp = KnowledgePostConverter.for_file(cls(), filename, format=format, postprocessors=postprocessors, interactive=interactive).from_file(filename, **opts) if src_paths: for src_path in src_paths: kp.add_srcfile(src_path) return kp @classmethod - def from_string(cls, string, src_strings={}, format=None, postprocessors=None, **opts): - kp = KnowledgePostConverter.for_format(cls(), format=format, postprocessors=postprocessors).from_string(string, ** opts) + def from_string(cls, string, src_strings={}, format=None, postprocessors=None, interactive=False, **opts): + kp = KnowledgePostConverter.for_format(cls(), format=format, postprocessors=postprocessors, interactive=interactive).from_string(string, ** opts) if src_strings: for src_name, data in list(src_strings.items()): kp.write_src(src_name, data) return kp - def to_file(self, filename, format=None, **opts): - return KnowledgePostConverter.for_file(self, filename, format=format).to_file(filename, **opts) + def to_file(self, filename, format=None, interactive=False, **opts): + return KnowledgePostConverter.for_file(self, filename, format=format, interactive=interactive).to_file(filename, **opts) - def to_string(self, format, **opts): - return KnowledgePostConverter.for_format(self, format).to_string(**opts) + def to_string(self, format, interactive=False, **opts): + return KnowledgePostConverter.for_format(self, format, interactive=interactive).to_string(**opts) from .converter import KnowledgePostConverter # noqa diff --git a/knowledge_repo/postprocessors/format_checks.py b/knowledge_repo/postprocessors/format_checks.py index f80dfc695..07328378b 100644 --- a/knowledge_repo/postprocessors/format_checks.py +++ b/knowledge_repo/postprocessors/format_checks.py @@ -3,36 +3,21 @@ import six from past.builtins import basestring +from ..post import HEADER_OPTIONAL_FIELD_TYPES, HEADER_REQUIRED_FIELD_TYPES from ..postprocessor import KnowledgePostProcessor -REQUIRED_FIELD_TYPES = { - 'title': six.string_types, - 'authors': list, - 'created_at': datetime.datetime, - 'tldr': six.string_types, - 'tags': list -} - -OPTIONAL_FIELD_TYPES = { - 'path': six.string_types, - 'updated_at': datetime.datetime, - 'private': bool, # If true, this post starts out private - 'allowed_groups': list, - 'thumbnail': (int, ) + six.string_types -} - class FormatChecks(KnowledgePostProcessor): _registry_keys = ['format_checks'] def process(self, kp): headers = kp.headers - for field, typ in REQUIRED_FIELD_TYPES.items(): + for field, typ, input in HEADER_REQUIRED_FIELD_TYPES: assert field in headers, "Required field `{}` missing from headers.".format( field) assert isinstance(headers[field], typ), "Value for field `{}` is of type {}, and needs to be of type {}.".format( field, type(headers[field]), typ) - for field, typ in OPTIONAL_FIELD_TYPES.items(): + for field, typ, input in HEADER_OPTIONAL_FIELD_TYPES: if field in headers: assert isinstance(headers[field], typ), "Value for field `{}` is of type {}, and needs to be of type {}.".format( field, type(headers[field]), typ)