Democratize: See in Helpful all usage examples of libraries (Emacs package)

(use-package democratize
  :demand t
  :config
  (democratize-enable-examples-in-helpful)
  (democratize-enable-examples-in-help))

(require 'democratize)
(democratize-enable-examples-in-helpful)
(democratize-enable-examples-in-help)

;;; democratize.el --- See in Helpful all usage examples of libraries -*- lexical-binding: t -*-
;;; Commentary:
;;;; For all the details, please do see the README
;;; Acknowledgments:
;;; Code:
;;;; Libraries
;;;; Symbols from other packages
;;;; Package metadata
;;;; Customizable variables
;;;; Other variables
;;;; Functions
;;;;; Description macro
;;;;; Utilities
;;;;;; Headings-related
;;;;;; Search symbols in extracted files
;;;;;; List files, libraries, and symbols
;;;;;;; files
;;;;;;; libraries
;;;;;;; symbols
;;;;;; Count symbols and examples
;;;;;; Make hash table of examples
;;;;;; Syntax highlighting
;;;;; Extract examples and create Org strings
;;;;;; xht-like
;;;;;;; xht
;;;;;; dash-like
;;;;;;; dash
;;;;;;; s
;;;;;; f-like
;;;;;;; f
;;;;;; shortdoc-based
;;;;;;; shortdoc
;;;;;;; treesit
;;;;; Democratize libraries
;;;;; Show overview of democratized library's functions
;;;;; Show stats
;;;;; Interface with helpful and help-fns
;;;;;; Enabling examples in help-like buffers
;;;;;;; helpful
;;;;;;; help-fns
;;;;;; Selecting a function to show help and examples for
;;;;;;; helpful
;;;;;;; help-fns
;;;;; See README
;;;;; See News
;;;; Wrapping up
;;; democratize.el ends here

;;; democratize.el --- See in Helpful all usage examples of libraries -*- lexical-binding: t -*-

;; SPDX-FileCopyrightText:  © flandrew <https://keyoxide.org/191F5B3E212EF1E515C19918AF32B9A5C1CCCB2D>
;; SPDX-License-Identifier: GPL-3.0-or-later

;;---------------------------------------------------------------------------
;; Author:            flandrew
;; Created:           2023-06-05
;; Version:           0.3.1
;; Homepage:          <https://flandrew.srht.site/listful/software.html>
;; Keywords:          lisp, docs
;; Package-Requires:  ((emacs "25.1")  (dash "2.14")  (s "1.12")  (f "0.20")  (xht "1.0.5"))
;;---------------------------------------------------------------------------

;; This file is part of Democratize, which is NOT part of GNU Emacs.

;; This program is free software: you can redistribute it and/or modify it
;; under the terms of the GNU General Public License as published by the Free
;; Software Foundation, either version 3 of the License, or (at your option)
;; any later version.
;;
;; This program is distributed in the hope that it will be useful, but WITHOUT
;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
;; FITNESS FOR A PARTICULAR PURPOSE. For more details, see the full license at
;; either LICENSES/GPL-3.0-or-later.txt or <https://www.gnu.org/licenses/>.

;;; Commentary:
;;
;; Democratize can extract thousands of usage examples (aka "demos") from some
;; of your favorite Emacs Lisp libraries and, among other things, insert them
;; into Helpful buffers when you look up a function.
;;
;; These examples come from files that are made available by the very
;; libraries. Alas, they often end up being overlooked for various reasons.
;; Overlook them no more: Democratize makes them easily accessible.
;;
;;;; For all the details, please do see the README
;;
;; Open it easily with any of the below:
;;   (find-file-read-only              "README.org")   <--- C-x C-e here¹, or
;;   (find-file-read-only-other-frame  "README.org")   <--- C-x C-e here¹, or
;;   (find-file-read-only-other-window "README.org")   <--- C-x C-e here¹
;;
;; or from any buffer:
;;   M-x democratize-see-readme
;;
;; or read it online:
;;   <https://flandrew.srht.site/listful/sw-emacs-democratize.html>
;;
;; ¹ or the key that ‘eval-last-sexp’ is bound to, if not C-x C-e.
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;;; Acknowledgments:
;;
;; Democratize was inspired by elisp-demos, from which it borrowed and
;; modified some functions, namely:
;;
;;  | function democratize-X              | used code from elisp-demos-Y  |
;;  |-------------------------------------+-------------------------------|
;;  | -syntax-highlight                   | -syntax-highlight             |
;;  | -search-symbol-in-file              | -search                       |
;;  | -list-symbols-in-file               | -symbols                      |
;;  | find-examples                       | find-demo                     |
;;  | help-find-examples-at-point         | help-find-demo-at-point       |
;;  | help-keymap                         | help-keymap                   |
;;  | find-examples-for-helpful-buffer    | for-helpful                   |
;;  | insert-examples-into-helpful-buffer | advice-helpful-update         |
;;  | insert-examples-into-help-buffer    | advice-describe-function-1    |
;;
;; About elisp-demos:
;;   SPDX-FileCopyrightText:  © 2018 Xu Chunyang
;;   SPDX-License-Identifier: GPL-3.0-or-later
;;   Author:                  Xu Chunyang
;;   Homepage:                <https://github.com/xuchunyang/elisp-demos>
;;
;; ------------------------------------------------------------------------
;;
;; The logic behind ‘democratize--str-of-shortdoc-pairs’ was built by
;; studying the shortdoc format itself, as specified by the shortdoc.el
;; native library.
;;
;; ------------------------------------------------------------------------
;;
;; Functions ‘democratize--describe’ and ‘democratize--examples-ht-dash-like’
;; were borrowed from, respectively, ‘orgreadme-fy-describe’ and
;; ‘orgreadme-fy-examples-ht’.
;;
;; About orgreadme-fy:
;;   SPDX-FileCopyrightText:  © flandrew
;;   SPDX-License-Identifier: GPL-3.0-or-later
;;   Author:                  flandrew
;;   Homepage:                <https://sr.ht/~flandrew/orgreadme-fy>
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;



;;; Code:
;;;; Libraries

(require 's)
(require 'f)
(require 'rx)
(require 'xht)       ; <---also by the author of this package
(require 'dash)
(require 'lisp-mnt)  ; ‘lm-summary’, ‘lm-version’, ‘lm-homepage’


;;;; Symbols from other packages

;; Silence "not known to be defined" compiler warnings
(defvar helpful--sym)
(declare-function helpful-callable               "ext:helpful")
(declare-function helpful--heading               "ext:helpful")
(declare-function outline-next-heading           "ext:outline")
(declare-function outline-hide-subtree           "ext:outline")
(declare-function outline-show-branches          "ext:outline")
(declare-function org-babel-goto-named-src-block "ext:ob-core")
(declare-function org-babel-get-src-block-info   "ext:ob-core")
(declare-function org-babel-src-block-names      "ext:ob-core")
(declare-function org-heading-components         "ext:org")
(declare-function org-narrow-to-subtree          "ext:org")
(declare-function org-mark-ring-push             "ext:org")
(declare-function org-show-entry                 "ext:org")


;;;; Package metadata

(defconst democratize--name "Democratize")

(defconst democratize--dot-el
  (format "%s.el" (file-name-sans-extension (eval-and-compile
                                              (or load-file-name
                                                  buffer-file-name)))))
(defconst democratize--readme-org
  (expand-file-name "README.org" (file-name-directory democratize--dot-el)))

(defconst democratize--summary  (lm-summary  democratize--dot-el))
(defconst democratize--version  (lm-version  democratize--dot-el))
(defconst democratize--homepage (lm-homepage democratize--dot-el))


;;;; Customizable variables

(defgroup democratize nil
  (format "%s." democratize--summary)
  :group 'docs
  :group 'lisp
  :link  '(emacs-library-link :tag "Lisp file" "democratize.el")
  :link  `(file-link :tag "README.org" ,democratize--readme-org)
  :link  `(url-link  :tag "Homepage"   ,democratize--homepage))

(defcustom democratize-title-for-helpful "Examples"
  "What Helpful will show as title for the function's demos/examples."
  :group 'democratize
  :type  'string)


;;;; Other variables

(defconst democratize--etc-dir
  (or (bound-and-true-p no-littering-etc-directory)
      (expand-file-name (convert-standard-filename "etc/")
                        user-emacs-directory))
  "Base etc/ dir to use.")

(defconst democratize--base-dir
  (expand-file-name (convert-standard-filename "democratize/")
                    democratize--etc-dir)
  "Base etc/democratize dir to use.")

(defconst democratize--extracted-dir
  (expand-file-name (convert-standard-filename "extracted/")
                    democratize--base-dir)
  "Subdirectory to store extracted demos/examples.")

(defconst democratize--downloads-dir
  (expand-file-name (convert-standard-filename "downloads/")
                    democratize--base-dir)
  "Subdirectory to store downloaded files.")

(defvar democratize--able-libraries '(xht dash s f shortdoc treesit)
  "List of libraries we're currently able to democratize.")

(defvar democratize--examples-source-raw-links
  (let* ((savannah   "https://git.savannah.gnu.org/")
         (plain-lisp (concat savannah "cgit/emacs.git/plain/lisp/")))
    (cdr
     `(:links
       xht  "https://git.sr.ht/~flandrew/xht/blob/master/README.org"
       dash "https://github.com/magnars/dash.el/raw/master/dev/examples.el"
       s    "https://github.com/magnars/s.el/raw/master/dev/examples.el"
       f    "https://github.com/rejeep/f.el/raw/master/README.org"
       shortdoc ,(concat plain-lisp "emacs-lisp/shortdoc.el")
       treesit  ,(concat plain-lisp "treesit.el"))))
  "Raw links to main source of usage examples of democratizable libraries.
It's a plist where keys are symbols representing the libraries and
values are the raw links.")

(defvar democratize--extractions-buffer "*democratize extractions*"
  "Name of buffer for extractions.")

(defvar democratize--downloads-buffer "*democratize downloads*"
  "Name of buffer for downloads.")

(defvar democratize--overview-buffer "*democratize overview*"
  "Name of buffer for overview.")

(defvar democratize--stats-buffer "*democratize stats*"
  "Name of buffer for stats.")


;;;; Functions
;;;;; Description macro

(defmacro democratize--describe (str)
  "Describe with string STR the contents under this heading.
Think of it as a docstring for the headings of your elisp files.

For the full docstring, look for ‘orgreadme-fy-describe’ in the
package ‘orgreadme-fy’."
  (declare (indent 0))
  (unless (stringp str)
    (user-error "‘democratize--describe’ must receive a string")))


;;;;; Utilities
;;;;;; Headings-related

(defun democratize--is-leaf-p (&optional pos)
  "Are we in an org heading and is it a leaf?
If POS is non-nil, answer this after going to POS. If POS is nil,
use point."
  (save-match-data
    (save-excursion
      (and pos (natnump pos) (goto-char pos))
      (beginning-of-line)
      (when (looking-at "^\\*+ ")
        (forward-line 1)
        (not (looking-at (regexp-quote (concat "*" (match-string 0)))))))))

(defun democratize--heading-contents ()
  "Return contents of this org heading."
  (save-excursion
    (forward-line 1)
    (s-trim (buffer-substring-no-properties
             (point) (if (re-search-forward "^\\*" nil t)
                         (line-beginning-position)
                       (point-max))))))

;;;;;; Search symbols in extracted files

(defun democratize--search-symbol-in-file (symbol file)
  "Search SYMBOL in FILE and return its examples or nil.
FILE is expected to be an Org file whose headings are strings
corresponding to function symbols, and whose contents are elisp org
src blocks with demos/examples for that function."
  (with-temp-buffer
    (insert-file-contents file)
    (goto-char (point-min))
    (save-match-data
      (when (re-search-forward
             (format "^\\* %s$" (regexp-quote (symbol-name symbol)))
             nil t)
        (save-excursion (democratize--heading-contents))))))

(defun democratize--search-symbol-in-files (symbol files)
  "Search SYMBOL in FILES and return its examples or nil.
FILES are expected to be a list of Org files whose headings are
strings corresponding to function symbols, and whose contents are
elisp org src blocks with demos/examples for that function.

If SYMBOL is found in one of the files, the search stops and the
examples found in the heading are returned."
  (let ((orgs (-copy files))
        res)
    (while (and orgs (not res))
      (--when-let (democratize--search-symbol-in-file symbol (pop orgs))
        (setq res it)))
    res))

(defun democratize--search-symbol-in-extracted-files (symbol)
  "Search SYMBOL in extracted files and return its examples or nil.
If SYMBOL is found in one of the files, the search stops and the
examples found in the heading are returned."
  (democratize--search-symbol-in-files
   symbol (democratize--list-extracted-files)))

(defun democratize--search-file-that-has-symbol (symbol files)
  "Search SYMBOL in FILES and return the file that has it.
FILES are expected to be a list of Org files whose headings are
strings corresponding to function symbols, and whose contents are
elisp org src blocks with demos/examples for that function.

If SYMBOL is found in one of the files, the search stops and the
filename that has it is returned."
  (let ((orgs (-copy files))
        res)
    (while (and orgs (not res))
      (--when-let (pop orgs)
        (when (democratize--search-symbol-in-file symbol it)
          (setq res (abbreviate-file-name it)))))
    res))


;;;;;; List files, libraries, and symbols
;;;;;;; files

(defun democratize--list-extracted-files ()
  "List all files having extracted examples."
  (let ((dir democratize--extracted-dir))
    (when (f-exists? dir)
      (f-files dir))))

;;;;;;; libraries

(defun democratize--list-democratized-libraries ()
  "List democratized libraries."
  (-map (-compose #'intern #'f-base)
        (democratize--list-extracted-files)))

(defun democratize--list-non-democratized-libraries ()
  "List libraries not yet democratized."
  (-difference democratize--able-libraries
               (democratize--list-democratized-libraries)))

(defun democratize--list-libraries-by-presence-of-examples ()
  "List democratizable libraries by presence of examples file.
It's a list of two lists: the first has democratizable libraries
for which a local file that is a source of examples is present; and
the second those for which it's not present."
  (let ((libs democratize--able-libraries)
        (pre "democratize--presence-of-examples"))
    (--separate (funcall (intern (format "%s-%s" pre it)))
                libs)))

(defun democratize--list-libraries-where-examples-present ()
  "List democratizable libraries for which examples file is present."
  (nth 0 (democratize--list-libraries-by-presence-of-examples)))

(defun democratize--list-libraries-where-examples-absent ()
  "List democratizable libraries for which examples file is absent."
  (nth 1 (democratize--list-libraries-by-presence-of-examples)))

;;;;;;; symbols

(defun democratize--list-symbols-in-file (file)
  "List all symbols in examples FILE."
  (with-temp-buffer
    (insert-file-contents file)
    (goto-char (point-min))
    (let (symbols)
      (while (re-search-forward "^\\* \\(.+\\)$" nil t)
        (push (intern (match-string-no-properties 1)) symbols))
      (nreverse symbols))))

(defun democratize--list-symbols ()
  "List all symbols from which we have extracted examples."
  (-flatten (-map #'democratize--list-symbols-in-file
                  (democratize--list-extracted-files))))


;;;;;; Count symbols and examples

(defun democratize--count-symbols-in-file (file)
  "Count all symbols in examples FILE."
  (when (f-exists? file)
    (length (democratize--list-symbols-in-file file))))

(defun democratize--count-examples-in-file (file)
  "Count all examples (demos) in examples FILE."
  (when (f-exists? file)
    (with-temp-buffer
      (insert-file-contents file)
      (how-many "=>\\|!!>\\|~>"))))


;;;;;; Make hash table of examples

(defun democratize--examples-ht-of-democratized-library (library)
  "Hash table whose keys are functions and values examples src blocks.
LIBRARY is a democratized library."
  (let* (exmp-ht
         (hdng-re "^\\*+ +\\([^ \n\r]+\\)")
         (libsym  (h-as-symbol library))
         (extract (f-expand (format "%s.org" library)
                            democratize--extracted-dir)))
    (unless (memq libsym (democratize--list-democratized-libraries))
      (error "Library %s has not been democratized" library))
    (unless (f-exists? extract)
      (error "File %s has not been found" extract))
    (with-temp-buffer
      (save-match-data
        (save-excursion (insert-file-contents extract))
        (setq exmp-ht (h-new (count-matches hdng-re)))
        (while (re-search-forward hdng-re nil t)
          (let ((k (h-as-symbol (match-string 1)))
                (v (save-excursion (democratize--heading-contents))))
            (h-put! exmp-ht k v))))
      exmp-ht)))


;;;;;; Syntax highlighting

(defun democratize--syntax-highlight (orgsrc)
  "Syntax-highlight ORGSRC."
  (with-temp-buffer
    (insert orgsrc)
    (delay-mode-hooks (org-mode))
    (if (fboundp 'font-lock-ensure)
        (font-lock-ensure)
      (with-no-warnings
        (font-lock-fontify-buffer)))
    (buffer-string)))


;;;;; Extract examples and create Org strings

(defun democratize--presence-of-examples (library file &optional subdir)
  "Check local presence of source of examples for LIBRARY.
If found, return the path. Otherwise return nil.

It's first checked in ‘democratize--downloads-dir’. If nothing
there, then the library's dir.

FILE is a string — the filename of the main source from which we
wish to extract demos/examples.

SUBDIR is the possible subdirectory in which it's located in the
library's repository. For example, dev in the case of ‘dash’,
where examples.el is located, which is the source we use for
democratizing that library.

If SUBDIR is nil or the empty string, the repo's root is assumed."
  (let (dld-dir exmp exmpz)
    (setq dld-dir (f-expand library democratize--downloads-dir)
          exmp    (f-expand file dld-dir)
          exmpz   (democratize--add-or-rem-gz exmp))
    (or (democratize--file-presence exmp)
        (democratize--file-presence exmpz)
        (progn
          (setq exmp  (-some->> (locate-library library)  f-parent
                                (f-expand (or (s-presence subdir) "."))
                                (f-expand file))
                exmpz (democratize--add-or-rem-gz exmp))
          (or (democratize--file-presence exmp)
              (democratize--file-presence exmpz))))))

(defun democratize--file-presence (&optional file)
  "If FILE exists, return FILE."
  (and file (f-exists? file) file))

(defun democratize--add-or-rem-gz (exmp)
  "Add or remove .gz extension to or from EXMP file."
  (when exmp
    (if (equal "gz" (f-ext exmp))
        (s-chop-suffix ".gz" exmp)
      (format "%s.gz" exmp))))

(defun democratize--not-found-error (library file &optional subdir)
  "Buffer with error message for LIBRARY's FILE not found.
FILE is a string — the filename of the main source from which we
wish to extract demos/examples.

SUBDIR is the possible subdirectory in which it's located in the
library's repository. For example, dev in the case of ‘dash’,
where examples.el is located, which is the source we use for
democratizing that library.

If SUBDIR is nil or the empty string, the repo's root is assumed."
  (let* ((dld-dir   (f-short (f-expand (h-as-string library)
                                       democratize--downloads-dir)))
         (raw-link  (plist-get democratize--examples-source-raw-links
                               (h-as-symbol library)))
         (error-msg (concat "The " file " of " library " has not been found: "
                            "cannot extract examples."))
         (full-msg  (concat
                     "\n" error-msg "\n\n"
                     "It must exist either in "
                     (if (s-presence subdir)
                         (format "subdir %s of the " subdir)
                       "the same ")
                     "directory that " library " was\n"
                     "loaded from or in " dld-dir "/ —\n"
                     "so put it there and try again.\n\n"
                     "You can fetch the raw latest version of the file "
                     "directly\nfrom that library's repository:\n\n  "
                     raw-link "\n\n"
                     "(Note: if you use git to fetch libraries, then "
                     "you probably\n"
                     " already have " file ", and could just symlink it to\n "
                     dld-dir "/ to make it available.)\n"))
         (buf       (get-buffer-create democratize--extractions-buffer)))
    (mkdir dld-dir '-p)
    (with-current-buffer buf
      (save-excursion
        (let ((inhibit-read-only t))
          (unless (zerop (buffer-size))
            (goto-char (point-min)) (insert "\n\f\n")
            (goto-char (point-min)))
          (insert (s-replace-regexp "^" "  " full-msg))
          (text-mode)
          (read-only-mode)))
      (switch-to-buffer buf)
      (error error-msg))))

(defun democratize--ht-to-org-str (hash-table)
  "Given a HASH-TABLE, print Org string with emacs-lisp source blocks.
HASH-TABLE's keys should be function names, and values should be
usage examples."
  (with-output-to-string
    (terpri)
    (h--each hash-table
      (princ (format "* %s\n\n%s\n%s\n%s\n\n"
                     key   "#+begin_src emacs-lisp"
                     value "#+end_src")))
    (terpri)))

(defun democratize--write-examples (string library)
  "Write STRING to examples Org file for LIBRARY."
  (mkdir democratize--extracted-dir '-p)
  ;; ‘f-write’ returns nil if successful, error otherwise.
  (f-write string 'utf-8
           (f-expand (format "%s.org" library)
                     democratize--extracted-dir))
  (message "%s has been successfully democratized!" library))

(defun democratize--overview-org-str-from-readme-org (library file heading)
  "Create org string for overview of LIBRARY's functions.
FILE is the full path to a README.org.

HEADING is a string representing the parent heading under which
function examples can be found."
  (unless (file-exists-p file)
    (error "File %s doesn't exist" file))
  (with-temp-buffer
    (insert-file-contents file)
    (let* ((case-fold-search t)
           (hdn heading)
           (hrx (format "^[*]+ %s" hdn))
           (pos (save-match-data
                  (goto-char (point-max))
                  (re-search-backward hrx nil t))))
      (unless pos
        (error "It seems heading %s doesn't exist in %s" hdn file))
      (autoload #'org-narrow-to-subtree "org")
      (org-narrow-to-subtree)
      (save-match-data
        (goto-char (point-min))
        (when (re-search-forward hrx nil t)
          (replace-match (format "* %s" library))))
      (keep-lines "^[*]+ ")
      (buffer-string))))

(defun democratize--overview (string)
  "Create overview buffer from org STRING."
  (with-output-to-temp-buffer democratize--overview-buffer
    (princ string))
  (with-current-buffer democratize--overview-buffer
    (when (fboundp 'org-mode)
      (org-mode)
      (outline-show-branches))))


;;;;;; xht-like

(defun democratize--presence-of-examples-xht-like (library)
  "Check local presence of source of examples for xht-like LIBRARY.
If found, return the path. Otherwise return nil."
  (democratize--presence-of-examples library "README.org"))

(defun democratize--not-found-error-xht-like (library)
  "Error message for xht-like LIBRARY's README.org not found."
  (democratize--not-found-error library "README.org"))

;; TODO: How to silence the unnecessary "position saved to mark-ring"
;;       message that comes up here? It seems to be coming from
;;       ‘org-mark-ring-push’, called by ‘org-babel-goto-named-src-block’.
(defun democratize--examples-ht-xht-like (&optional file prefix)
  "Extract all examples from xht-like library. Return hash table.
FILE is the path to a xht-like library's README.org file. If nil,
use current buffer.

A xht-like library is one whose README.org file contains Emacs Lisp
org src blocks named PREFIX-function-name. When nil, PREFIX
defaults to \"examples\".

So, for instance, xht's README.org contains an org src block that
begins with:
  #+name: examples-h-new
inside which one finds examples for the function ‘h-new’."
  (when file
    (unless (file-exists-p file)
      (error "File %s doesn't exist" file)))
  (require 'ob-core)
  (autoload #'org-mark-ring-push "org") ;<-- ‘org-babel-goto-named-src-block’
  (let ((contents (if file
                      (f-read file)
                    (buffer-substring-no-properties
                     (point-min) (point-max))))
        block-names  prefix-  functions  examples-str)
    (with-temp-buffer
      (insert contents)
      (setq block-names  (reverse (org-babel-src-block-names))
            prefix-      (format "%s-" (or prefix "examples"))
            functions    (--map (s-chop-prefix prefix- it)
                                block-names)
            examples-str (--map (-> it
                                    org-babel-goto-named-src-block
                                    org-babel-get-src-block-info
                                    cadr)
                                block-names))
      (h-zip-lists functions examples-str))))

(defun democratize--org-str-of-examples-xht-like (&optional file prefix)
  "Extract all examples from xht-like library. Return Org string.
A xht-like library is one that has a README.org FILE containing
sexps defining examples, such as ‘xht’.

For the meaning of FILE and PREFIX, see docstring of
‘democratize--examples-ht-xht-like’."
  (democratize--ht-to-org-str (democratize--examples-ht-xht-like
                               file prefix)))

(defun democratize--org-str-of-examples-xht-like-maybe (library)
  "Extract all examples from xht-like LIBRARY. Return Org string."
  (let ((examples (democratize--presence-of-examples-xht-like library))
        pr)
    (unless examples
      (democratize--not-found-error-xht-like library))
    (setq pr (make-progress-reporter
              (format "Extracting usage examples from %s ... " library)))
    (prog1 (democratize--org-str-of-examples-xht-like examples "examples")
      (progress-reporter-done pr))))

(defun democratize--overview-xht-like-org-str (library &optional heading)
  "Create org string for overview of xht-like LIBRARY's functions.
HEADING is a string representing the parent heading under which
function examples can be found. If nil, it defaults to the one
currently used by ‘xht’, namely \"Functions\"."
  (democratize--overview-org-str-from-readme-org
   library
   (democratize--presence-of-examples-xht-like library)
   (or heading "Functions")))

(defun democratize--overview-xht-like (library &optional heading)
  "Show overview of xht-like LIBRARY's functions.
See docstring of ‘democratize--overview-xht-like-org-str’ for the
meaning of HEADING."
  (democratize--overview
   (democratize--overview-xht-like-org-str library heading)))


;;;;;;; xht

(defun democratize--presence-of-examples-xht ()
  "Check local presence of source of examples for ‘xht’.
If found, return the path. Otherwise return nil."
  (democratize--presence-of-examples-xht-like "xht"))

(defun democratize--not-found-error-xht ()
  "Error message for xht's README.org not found."
  (democratize--not-found-error-xht-like "xht"))

(defun democratize--org-str-of-examples-xht ()
  "Extract all examples from ‘xht’. Return Org string."
  (democratize--org-str-of-examples-xht-like-maybe "xht"))

(defun democratize--write-examples-xht ()
  "Extract all examples from ‘xht’ and write them to an Org file."
  (democratize--write-examples
   (democratize--org-str-of-examples-xht) "xht"))

(defun democratize--overview-xht ()
  "Show overview of ‘xht’'s functions."
  (democratize--overview-xht-like "xht"))


;;;;;; dash-like

(defun democratize--presence-of-examples-dash-like (library)
  "Check local presence of source of examples for dash-like LIBRARY.
If found, return the path. Otherwise return nil."
  (democratize--presence-of-examples library "examples.el" "dev"))

(defun democratize--not-found-error-dash-like (library)
  "Error message for dash-like LIBRARY's examples.el not found."
  (democratize--not-found-error library "examples.el" "dev"))

(defun democratize--strip-defexamples-dash-like (string)
  "Strip defexamples from STRING and remove indentation."
  (s-with string
    (s-replace-regexp "\\`.*\n+" "")
    (s-replace-regexp "\n*)\\'"  "")
    (s-replace-regexp "^    "    "")))

(defun democratize--examples-ht-dash-like (file &optional defname)
  "Given examples FILE, return hash table with sexp strings of examples.
Optional argument DEFNAME is the function used to define examples.
If nil, defaults to ‘defexamples’, which is the one used by ‘dash’
and ‘s’.

Whatever comes after DEFNAME will be the key.

The value will be the whole sexp as string. No change in formatting
happens: treated as string, not sexp — so it preserves the original
indentation and alignment."
  (unless (file-exists-p file)
    (error "File %s doesn't exist" file))
  (let* ((result  (h-new))
         (defname (if defname
                      (h-as-string defname)
                    "defexamples"))
         (rxf     (eval `(rx (: bol (* blank) "(" (| ,defname)
                                (+ (any " \t\n"))
                                (group (+ (not (any " \t\n")))))))))
    (with-temp-buffer
      (insert-file-contents file)
      (with-syntax-table emacs-lisp-mode-syntax-table
        (while (re-search-forward rxf nil t)
          (beginning-of-line)
          (h-put! result
                  (match-string 1)
                  (democratize--strip-defexamples-dash-like
                   (buffer-substring-no-properties (point)
                                                   (progn
                                                     (forward-sexp)
                                                     (point))))))
        result))))

(defun democratize--org-str-of-examples-dash-like (file &optional defname)
  "Extract all examples from dash-like library. Return Org string.
A dash-like library is one that has an examples.el FILE containing
sexps defining examples, such as ‘dash’ and ‘s’.

For the meaning of FILE and DEFNAME, see docstring of
‘democratize--examples-ht-dash-like’."
  (democratize--ht-to-org-str (democratize--examples-ht-dash-like
                               file defname)))

(defun democratize--org-str-of-examples-dash-like-maybe (library)
  "Extract all examples from dash-like LIBRARY. Return Org string."
  (let ((examples (democratize--presence-of-examples-dash-like library))
        pr)
    (unless examples
      (democratize--not-found-error-dash-like library))
    (setq pr (make-progress-reporter
              (format "Extracting usage examples from %s ... " library)))
    (prog1
        (democratize--org-str-of-examples-dash-like examples "defexamples")
      (progress-reporter-done pr))))

(defun democratize--overview-dash-like-org-str (library &optional
                                                        defname defgroup)
  "Create org string for overview of dash-like LIBRARY's functions.
- Optional argument DEFNAME is the function used to define
  examples. If nil, defaults to ‘defexamples’, which is the one
  used by ‘dash’ and ‘s’.

- Optional argument DEFGROUP is the function used to define
  examples group. If nil, defaults to ‘def-example-group’, which is
  the one used by ‘dash’ and ‘s’."
  (let ((file (democratize--presence-of-examples-dash-like library))
        def  deg  drx)
    (unless (file-exists-p file)
      (error "File %s doesn't exist" file))
    (with-temp-buffer
      (insert-file-contents file)
      (setq def (if defname (h-as-string defname)  "defexamples")
            deg (if defname (h-as-string defgroup) "def-example-group")
            drx (eval `(rx (or ,(format "(%s " def)
                               ,(format "(%s " deg)))))
      (keep-lines drx)
      (->> (buffer-string)
           (s-replace-regexp drx  "")
           (s-replace-regexp "\"" "")
           (s-replace-regexp "^[^ ]" "** \\&")
           (s-replace-regexp "^[ ]+" "*** ")
           (format "* %s\n%s" library)))))

(defun democratize--overview-dash-like (library &optional defname defgroup)
  "Show overview of dash-like LIBRARY's functions.
See ‘democratize--overview-dash-like-org-str’ for meaning of
DEFNAME and DEFGROUP."
  (democratize--overview
   (democratize--overview-dash-like-org-str library defname defgroup)))


;;;;;;; dash

(defun democratize--presence-of-examples-dash ()
  "Check local presence of source of examples for ‘dash’.
If found, return the path. Otherwise return nil."
  (democratize--presence-of-examples-dash-like "dash"))

(defun democratize--not-found-error-dash ()
  "Error message for dash's examples.el not found."
  (democratize--not-found-error-dash-like "dash"))

(defun democratize--org-str-of-examples-dash ()
  "Extract all examples from ‘dash’. Return Org string."
  (democratize--org-str-of-examples-dash-like-maybe "dash"))

(defun democratize--write-examples-dash ()
  "Extract all examples from ‘dash’ and write them to an Org file."
  (democratize--write-examples
   (democratize--org-str-of-examples-dash) "dash"))

(defun democratize--overview-dash ()
  "Show overview of ‘dash’'s functions."
  (democratize--overview-dash-like "dash"))


;;;;;;; s

(defun democratize--presence-of-examples-s ()
  "Check local presence of source of examples for ‘s’.
If found, return the path. Otherwise return nil."
  (democratize--presence-of-examples-dash-like "s"))

(defun democratize--not-found-error-s ()
  "Error message for s's examples.el not found."
  (democratize--not-found-error-dash-like "s"))

(defun democratize--org-str-of-examples-s ()
  "Extract all examples from ‘s’. Return Org string."
  (democratize--org-str-of-examples-dash-like-maybe "s"))

(defun democratize--write-examples-s ()
  "Extract all examples from ‘s’ and write them to an Org file."
  (democratize--write-examples
   (democratize--org-str-of-examples-s) "s"))

(defun democratize--overview-s ()
  "Show overview of ‘s’'s functions."
  (democratize--overview-dash-like "s"))


;;;;;; f-like

(defun democratize--presence-of-examples-f-like (library)
  "Check local presence of source of examples for f-like LIBRARY.
If found, return the path. Otherwise return nil."
  (democratize--presence-of-examples library "README.org"))

(defun democratize--not-found-error-f-like (library)
  "Error message for f-like LIBRARY's README.md not found."
  (democratize--not-found-error library "README.org"))

(defun democratize--org-str-of-examples-f-like (file &optional heading prefix)
  "Extract all examples from f-like library. Return Org string.
FILE is the path to an f-like library's README.org file.

HEADING is a string representing the parent heading under which
function examples can be found. If nil, it defaults to the one
currently used by ‘f’, namely \"Documentation and examples\".

When nil, PREFIX defaults to \"examples\".

An f-like library is one whose README.org file contains Emacs Lisp
blocks with examples, and which doesn't have a larger
dev/examples.el (or equivalent) file from which examples could
preferably be extracted."
  (with-temp-buffer
    (insert-file-contents file)
    ;; The strategy here is to:
    ;; 1) Name all function-related source blocks as examples-FUNCTION, which
    ;;    would then make the .org file xht-like.
    ;; 2) Apply the appropriate xht-like functions to extract the data.
    ;; This is slightly inefficient, since it'll make two passes. But the
    ;; costs are barely perceptible, and the resulting code is simpler.
    (let* ((case-fold-search t)
           (hdn (or heading "Documentation and examples"))
           (pfx (or prefix  "examples"))
           (hrx (format "^[*]+ %s" hdn))
           (pos (save-match-data
                  (goto-char (point-max))
                  (re-search-backward hrx nil t))))
      (unless pos
        (error "It seems heading %s doesn't exist in %s" hdn file))
      (delay-mode-hooks (org-mode))
      (org-narrow-to-subtree)
      (save-match-data
        (goto-char (point-max))
        (while (re-search-backward "^..begin_src.*lisp" nil t)
          (forward-line -1)
          ;; So that if f-like libraries ever name their blocks like this,
          ;; democratize's code will continue working:
          (unless (looking-at (format "..name: %s-" pfx))
            (forward-line 1)
            (insert (format "#+name: %s-%s\n" pfx
                            ;; The heading string (also the function name):
                            (nth 4 (org-heading-components)))))))
      (democratize--ht-to-org-str
       (democratize--examples-ht-xht-like)))))

(defun democratize--org-str-of-examples-f-like-maybe (library)
  "Extract all examples from f-like LIBRARY. Return Org string."
  (let ((examples (democratize--presence-of-examples-f-like library))
        pr)
    (unless examples
      (democratize--not-found-error-f-like library))
    (setq pr (make-progress-reporter
              (format "Extracting usage examples from %s ... " library)))
    (prog1 (democratize--org-str-of-examples-f-like examples)
      (progress-reporter-done pr))))

(defun democratize--overview-f-like-org-str (library &optional heading)
  "Create org string for overview of f-like LIBRARY's functions.
HEADING is a string representing the parent heading under which
function examples can be found. If nil, it defaults to the one
currently used by ‘f’, namely \"Documentation and examples\"."
  (democratize--overview-org-str-from-readme-org
   library
   (democratize--presence-of-examples-f-like library)
   (or heading "Documentation and examples")))

(defun democratize--overview-f-like (library &optional heading)
  "Show overview of f-like LIBRARY's functions.
See ‘democratize--overview-f-like-org-str’ for meaning of HEADING."
  (democratize--overview
   (democratize--overview-f-like-org-str library heading)))


;;;;;;; f

(defun democratize--presence-of-examples-f ()
  "Check local presence of source of examples for ‘f’.
If found, return the path. Otherwise return nil."
  (democratize--presence-of-examples-f-like "f"))

(defun democratize--not-found-error-f ()
  "Error message for f's README.org not found."
  (democratize--not-found-error-f-like "f"))

(defun democratize--org-str-of-examples-f ()
  "Extract all examples from ‘f’. Return Org string."
  (democratize--org-str-of-examples-f-like-maybe "f"))

(defun democratize--write-examples-f ()
  "Extract all examples from ‘f’ and write them to an Org file."
  (democratize--write-examples
   (democratize--org-str-of-examples-f) "f"))

(defun democratize--overview-f ()
  "Show overview of ‘f’'s functions."
  (democratize--overview-f-like "f"))


;;;;;; shortdoc-based

;; functions directly used by the shortdoc-based libraries

(defun democratize--presence-of-examples-shortdoc-based (library
                                                         &optional file)
  "Check local presence of shortdoc FILE for LIBRARY.
If found, return the path. Otherwise return nil.
If FILE is nil, use LIBRARY's .el."
  (let ((f (democratize--shortdoc-file-given-lib library file)))
    (democratize--presence-of-examples library f)))

(defun democratize--not-found-error-shortdoc-based (library &optional file)
  "Error message for LIBRARY's shortdoc FILE not found.
If FILE is nil, use LIBRARY's .el."
  (let ((f (f-filename (democratize--shortdoc-file-given-lib library file))))
    (democratize--not-found-error library f)))

(defun democratize--org-str-of-examples-shortdoc-based-maybe (library
                                                              &optional file)
  "Extract all examples from shortdoc-group LIBRARY. Return Org string.
FILE is a string — the filename of the main source from which we
wish to extract demos/examples."
  (let ((examples (democratize--presence-of-examples-shortdoc-based
                   library file))
        pr)
    (unless examples
      (democratize--not-found-error-shortdoc-based library file))
    (setq pr (make-progress-reporter
              (format "Extracting usage examples from %s ... " library)))
    (prog1 (democratize--org-str-of-examples-shortdoc-based examples)
      (progress-reporter-done pr))))

(defun democratize--overview-shortdoc-based (library)
  "Show overview of shortdoc-based LIBRARY's functions."
  (democratize--overview
   (democratize--overview-org-str-from-shortdoc-library library)))

;; functions that support the above

(defun democratize--shortdoc-file-given-lib (library &optional file)
  "Given LIBRARY name and FILE, return shortdoc file.
Helper for ‘democratize--presence-of-examples-shortdoc-based’ and
‘democratize--not-found-error-shortdoc-based’."
  (-if-let* ((libfile (locate-library library)))
      (if file
          (-some->> libfile f-parent (f-expand file))
        (-some->> libfile f-no-ext f-no-ext (format "%s.el")))
    (if file file (format "%s.el" library))))

(defun democratize--org-str-of-examples-shortdoc-based (&optional file)
  "Output Org headings for shortdoc-groups–containing FILE.
FILE is the path containing ‘define-short-documentation-group’
sexps for a library. If nil, use current buffer."
  (democratize--org-str-of-shortdoc-groups
   (democratize--shortdoc-groups-ht file)))

(defun democratize--shortdoc-groups-ht (&optional file)
  "Make hash table with all short documentation groups in FILE.
FILE is the path containing ‘define-short-documentation-group’
sexps for a library. If nil, use current buffer.

The hash table has as keys the name of the group, and as values the
definition sexps themselves."
  (when file
    (unless (file-exists-p file)
      (error "File %s doesn't exist" file)))
  (let ((contents (if file
                      (with-temp-buffer
                        (insert-file-contents file)
                        ;; this ^ can deal with .el.gz
                        (buffer-string))
                    (buffer-substring-no-properties
                     (point-min) (point-max))))
        (dsdg-re  (rx (: "("
                         (* blank) "define-short-documentation-group"
                         (* blank))))
        groups-ht  sexp  mb)
    (with-temp-buffer
      (insert contents)
      (goto-char (point-min))
      (save-match-data
        (setq groups-ht (h-new (count-matches dsdg-re)))
        (while (re-search-forward dsdg-re nil t)
          (setq mb   (match-beginning 0)
                sexp (h-read-list (buffer-substring-no-properties
                                   mb (save-excursion
                                        (goto-char mb)
                                        (forward-list)
                                        (point)))))
          (h-put! groups-ht (cadr sexp) sexp))
        groups-ht))))

(defun democratize--value-to-string (value)
  "Convert VALUE to a string ready for printing.
VALUE appears in the output of ‘democratize--shortdoc-funs-ht’, and
it can be either a string or a list of strings depending on whether
more than one of the processed groups contained the same function."
  (->> value  -list  -uniq  nreverse  (s-join "")  s-chomp))

(defun democratize--org-str-of-shortdoc-groups (htbl)
  "Output Org headings for hash table HTBL of shortdoc groups.
HTBL is generated by ‘democratize--shortdoc-groups-ht’."
  (let ((beg "#+begin_src emacs-lisp")
        (end "#+end_src"))
    (with-output-to-string
      (h--each (democratize--shortdoc-funs-ht htbl)
        (let ((value (democratize--value-to-string value)))
          (unless (s-blank-str? value)
            (princ (format "* %s\n\n%s\n%s\n%s\n\n"
                           key beg value end))))))))

(defun democratize--shortdoc-funs-ht (htbl)
  "Create a functions hash table from HTBL of shortdoc groups.
HTBL is generated by ‘democratize--shortdoc-groups-ht’.

This hash table flattens the HTBL input, eliminating the groups.
Each key of the output is a function for which there's shortdoc,
and each value is a string representing its available examples."
  (let ((fun-ht (h-new))
        (groups (h-lvalues htbl)))
    (dolist (group groups fun-ht)
      (dolist (elt (-filter #'listp group))
        (-let [(fun . pairs) elt]
          (h-put-add! fun-ht fun
                      (democratize--str-of-shortdoc-pairs pairs)))))))

(defun democratize--ins-or-p1 (value buf)
  "Either ‘insert’ or ‘prin1’ VALUE in buffer BUF.
Helper for ‘democratize--str-of-shortdoc-pairs’."
  (if (stringp value)
      (insert value)
    (prin1 value buf)))

(defun democratize--str-of-shortdoc-pairs (pairs)
  "Create string of examples from shortdoc element's PAIRS.
PAIRS is the cdr of a shortdoc element — that is, the element
stripped of the function name that appears as car.

Example:
  (democratize--str-of-shortdoc-pairs
   \\='(:eval (+ 1 2)
     :eval (+ 20 22)))"
  (with-temp-buffer
    (let ((print-quoted t)
          (print-escape-newlines t)
          (eg   "  ; e.g.")
          (id   "[It depends]")  ; Capitalized to avoid dash fontification
          (cne  "[Could not eval]")
          (cbuf (current-buffer))
          key  value)
      (while pairs
        (setq key   (pop pairs)
              value (pop pairs))
        (pcase key
          ;; Print sexp and results -----------------------------------------
          (:eval             (democratize--ins-or-p1 value cbuf)
                             (insert "\n=> ")
                             (let ((v (condition-case nil
                                          (eval
                                           (if (stringp value)
                                               (car (read-from-string value))
                                             value))
                                        (error cne))))
                               (if (equal v cne)
                                   (insert v)
                                 (prin1 v cbuf))))
          (:no-eval*         (democratize--ins-or-p1 value cbuf)
                             (insert "\n=> " id))
          ;; Print sexp -----------------------------------------------------
          (:no-eval          (democratize--ins-or-p1 value cbuf))
          (:no-value         (democratize--ins-or-p1 value cbuf))
          ;; Print results --------------------------------------------------
          (:result           (insert "=> ") (prin1 value cbuf))
          (:result-string    (insert "=> ") (princ value cbuf))
          (:eg-result        (insert "=> ") (prin1 value cbuf) (insert eg))
          (:eg-result-string (insert "=> ") (princ value cbuf) (insert eg)))
        (pcase key
          ;; Print newline (maybe) ------------------------------------------
          (:no-manual (ignore))
          (:args      (ignore))
          (_          (insert "\n"))))
      (buffer-string))))

(defun democratize--overview-org-str-from-shortdoc-file (file)
  "Create org string for overview of FILE's functions.
FILE is the full path to a file containing shortdoc groups."
  (when file
    (unless (file-exists-p file)
      (error "File %s doesn't exist" file))
    (with-output-to-string
      (h--each (democratize--shortdoc-groups-ht file)
        (!cdr value)
        (princ (format "** %s\n" (pop value)))
        (pcase (-any? #'stringp value)
          ('nil (--each value
                  (princ (format "*** %s\n" (car it)))))
          (_    (--each value
                  (princ (if (stringp it)
                             (format "*** %s\n" it)
                           (format "**** %s\n" (car it)))))))))))

(defun democratize--overview-org-str-from-shortdoc-library (library)
  "Create org string for overview of shortdoc-based LIBRARY's functions."
  (format "* %s\n%s"
          library
          (democratize--overview-org-str-from-shortdoc-file
           (democratize--presence-of-examples-shortdoc-based library))))

;; unused but potentially useful

(defun democratize--str-of-shortdoc-elt (elt)
  "Create string of examples from shortdoc element ELT.
Example:
  (democratize--str-of-shortdoc-elt
   \\='(+
     :eval (+ 1 2 3)
     :eval (+ 20 22)))

Note: this function is unused, but added here as a utility for
testing and debugging."
  (democratize--str-of-shortdoc-pairs (cdr elt)))

(defun democratize--str-of-shortdoc-group (group)
  "Create string of examples from shortdoc GROUP.
Example:
  (democratize--str-of-shortdoc-group
    \\='(define-short-documentation-group \"math-operations\"
       \"Addition operations\"
       (+
        :eval (+ 2 3 5)
        :eval (+ 21 21))
       \"Multiplication operations\"
       (*
        :eval (* 2 3 5)
        :eval (* 21 2))))

Note: this function is unused, but added here as a utility for
testing and debugging."
  (let ((fun-ht (h-new)))
    (dolist (elt (-filter #'listp group))
      (-let [(fun . pairs) elt]
        (h-put-add! fun-ht fun
                    (democratize--str-of-shortdoc-pairs pairs))))
    (with-output-to-string
      (h--each fun-ht
        (princ (democratize--value-to-string value))
        (terpri)))))


;;;;;;; shortdoc

(defun democratize--presence-of-examples-shortdoc ()
  "Check local presence of shortdoc file for ‘shortdoc’.
If found, return the path. Otherwise return nil."
  (democratize--presence-of-examples-shortdoc-based "shortdoc"))

(defun democratize--not-found-error-shortdoc ()
  "Error message for shortdoc file for ‘shortdoc’ not found."
  (democratize--not-found-error-shortdoc-based "shortdoc"))

(defun democratize--org-str-of-examples-shortdoc ()
  "Extract all examples from ‘shortdoc’. Return Org string."
  (democratize--org-str-of-examples-shortdoc-based-maybe "shortdoc"))

(defun democratize--write-examples-shortdoc ()
  "Extract all examples from ‘shortdoc’ and write them to an Org file."
  (democratize--write-examples
   (democratize--org-str-of-examples-shortdoc) "shortdoc"))

(defun democratize--overview-shortdoc ()
  "Show overview of ‘shortdoc’'s functions."
  (democratize--overview-shortdoc-based "shortdoc"))


;;;;;;; treesit

(defun democratize--presence-of-examples-treesit ()
  "Check local presence of shortdoc file for ‘treesit’.
If found, return the path. Otherwise return nil."
  (democratize--presence-of-examples-shortdoc-based "treesit"))

(defun democratize--not-found-error-treesit ()
  "Error message for shortdoc file for ‘treesit’ not found."
  (democratize--not-found-error-shortdoc-based "treesit"))

(defun democratize--org-str-of-examples-treesit ()
  "Extract all examples from ‘treesit’. Return Org string."
  (democratize--org-str-of-examples-shortdoc-based-maybe "treesit"))

(defun democratize--write-examples-treesit ()
  "Extract all examples from ‘treesit’ and write them to an Org file."
  (democratize--write-examples
   (democratize--org-str-of-examples-treesit) "treesit"))

(defun democratize--overview-treesit ()
  "Show overview of ‘treesit’'s functions."
  (democratize--overview-shortdoc-based "treesit"))


;;;;; Democratize libraries

(democratize--describe
  "Commands related to the democratization of libraries.")

(defun democratize--select-library (libs error-msg read-msg format-str
                                         &optional library)
  "The arguments are:
- LIBS       -- a list of libraries
- ERROR-MSG  -- error message to show
- READ-MSG   -- completion-read message to show
- FORMAT-STR -- string to pass to format
- LIBRARY    -- input library

This is a helper for both ‘democratize-library’ and
‘democratize-overview-library’."
  (let ((lib (if library
                 (if (member (h-as-symbol library) libs)
                     library
                   (error error-msg library))
               (completing-read read-msg libs))))
    (->> lib (format format-str) intern funcall)))

;;;###autoload
(defun democratize-library (&optional library)
  "Select a library to democratize.

When running this function interactively, a list of democratizable
libraries will be shown. Select one to democratize. Alternatively,
you can pass LIBRARY to democratize it non-interactively.

That library has a chosen source file which is rich in examples. By
democratizing the library, you extract these examples into an Org
file that is saved in your Emacs etc directory. It'll then be
available for operations such as integration with Helpful buffers
when you look up a function from that library.

If a library cannot be democratized because its source of examples
isn't locally available, instructions will be given about how to
fetch it.

Redemocratizing a library is ok: it'll simply overwrite the .org file
previously created by this very function, thereby updating it."
  (interactive)
  (democratize--select-library
   democratize--able-libraries
   "Library %s isn't democratizable"
   "Select a library to democratize: "
   "democratize--write-examples-%s"
   library))

;;;###autoload
(defun democratize-all-libraries ()
  "Democratize all democratizable libraries.
To see a list, run ‘democratize-able-libraries’.

Some libraries might not be able to be democratized because their
source of examples is locally unavailable. These will be skipped,
and instructions will be given about how to fetch the sources.

For more information, see ‘democratize-library’."
  (interactive)
  (--each democratize--able-libraries
    (ignore-errors
      (funcall (intern (format "democratize--write-examples-%s" it)))))
  (let ((abs (democratize--list-libraries-where-examples-absent))
        (dex democratize--extractions-buffer)
        (ddl democratize--downloads-buffer)
        (dst democratize--stats-buffer))
    (when abs
      (democratize-show-commands-to-download-pending-examples)
      (switch-to-buffer dex))
    (democratize-show-stats)
    (when abs
      (with-current-buffer dst
        (goto-char (point-max))
        (let ((inhibit-read-only t))
          (insert "\n  Check buffer " dex " for more\n  "
                  "information about non-democratized libraries.\n"))
        (goto-char (point-min)))
      (with-current-buffer dex
        (goto-char (point-min))
        (let ((inhibit-read-only t))
          (insert "\n  Check buffer " ddl " for shell commands\n  "
                  "with which to download the pending files listed below.\n"
                  "\n\f\n"))
        (goto-char (point-min))))
    (switch-to-buffer dst)))

;;;###autoload
(defun democratize-able-libraries ()
  "List democratizable libraries."
  (interactive)
  (->> democratize--able-libraries
       (-map  #'symbol-name)
       (-sort #'string<)
       (s-join ", ")
       (message "Democratizable libraries: %s")))

;;;###autoload
(defun democratize-d-libraries ()
  "List democratized libraries."
  (interactive)
  (--if-let (democratize--list-democratized-libraries)
      (->> it
           (-map #'symbol-name)
           (s-join ", ")
           (message "Democratized libraries: %s"))
    (message "No libraries have been democratized yet.")))

;;;###autoload
(defun democratize-able-libraries-pending ()
  "List democratizable libraries not yet democratized."
  (interactive)
  (--if-let (democratize--list-non-democratized-libraries)
      (->> it
           (-map  #'symbol-name)
           (-sort #'string<)
           (s-join ", ")
           (message "Democratizable libraries not yet democratized: %s"))
    (message "There're no libraries pending democratization.")))

;;;###autoload
(defun democratize-show-commands-to-download-pending-examples ()
  "Show shell commands to download pending examples files.
These files are needed to democratize pending libraries."
  (interactive)
  (let ((absent (democratize--list-libraries-where-examples-absent)))
    (if (not absent)
        (message "There're no pending examples files to retrieve.")
      (let* ((all-strs (-map #'democratize--cmds-to-dld-library-examples
                             absent))
             (curl-str (->> all-strs  (-map #'car)  (s-join "\n")))
             (wget-str (->> all-strs  (-map #'cadr) (s-join "\n")))
             (buf (get-buffer-create democratize--downloads-buffer))
             (msg (concat "
# You can democratize all not-yet-democratized democratizable libraries by
# downloading pending examples files. Here are some shell commands for that:\n
# Using curl:\n" curl-str "\n
# Using wget:\n" wget-str "\n
# (Note that the correctness of the commands output above hasn't been tested
# in different systems. Check that all looks fine and properly escaped before
# using them. Or you can use some other download method of your preference.)\n
# Then try to:\n#   M-x democratize-all-libraries")))
        (with-current-buffer buf
          (erase-buffer)
          (insert msg)
          (goto-char (point-min))
          (ignore-errors (sh-mode))
          (read-only-mode))
        (switch-to-buffer buf)))))

(defun democratize--cmds-to-dld-library-examples (library)
  "Strings of commands that would download examples file of LIBRARY.
Result is a list of equivalent commands. The list has two elements:
the first, using curl; the second, using wget."
  (let* ((dld-dir (->> democratize--downloads-dir
                       (f-expand (h-as-string library))))
         (dld-dirq (-> dld-dir  shell-quote-argument  f-short))
         (raw-link (-> democratize--examples-source-raw-links
                       (plist-get (h-as-symbol library))))
         ;; Both would:
         ;; - follow redirects
         ;; - save using the remote name
         ;; - overwrite a previous copy of the file if it already existed.
         (curl-cmd "curl -L -O")
         (wget-cmd "wget -r") ;; -r = --recursive (implies "overwrite")
         (all-cmds `(,curl-cmd ,wget-cmd)))
    (mkdir dld-dir '-p)
    (--map (format "cd %s &&\n  %s \"%s\""
                   dld-dirq  it  raw-link)
           all-cmds)))


;;;;; Show overview of democratized library's functions

(democratize--describe
  "It's great to look up a library's function and see examples. But when we
keep coming back to the same trees, we may lose sight of the forest.

To what group does this function belong to?
What other functions are there in the library that you never remember to use?

The overview command shows you the library's forest, so to speak. Or more
precisely, it shows you a big Org tree with all functions for which there are
examples, nicely grouped in the categories defined by the library's author.")

;;;###autoload
(defun democratize-overview-library (&optional library)
  "Select a democratized LIBRARY to show overview of.
The overview is an Org tree with all functions for which there are
examples, grouped in the categories defined by the library's author.

When running this function interactively, a list of democratized
libraries will be shown. Pick one. Alternatively, you can pass
LIBRARY non-interactively.

Browse it. Pick one to look into. Put point on it. Then press the
handy key that you've chosen to bind ‘helpful-at-point’ to.

If you'd like the examples to be already included in the Org tree's
leaves, use ‘democratize-full-overview-library’ instead."
  (interactive)
  (democratize--select-library
   (democratize--list-democratized-libraries)
   "Library %s hasn't been democratized"
   "Select a democratized library to show overview of: "
   "democratize--overview-%s"
   library))

;;;###autoload
(defun democratize-full-overview-library (&optional library)
  "Select a democratized LIBRARY to show full overview of.
This is just like ‘democratize-overview-library’, but with the
examples source blocks included in the leaves."
  (interactive)
  (democratize-overview-library library)
  (with-current-buffer democratize--overview-buffer
    (let* ((inhibit-read-only t)
           (lib (or library (buffer-substring-no-properties
                             (+ 2 (point)) (line-end-position))))
           (exmp-ht (democratize--examples-ht-of-democratized-library lib))
           (hdng-re "^\\*+ +\\([^ \n\r]+\\)"))
      (save-match-data
        (while (not (eobp))
          (and (democratize--is-leaf-p)
               (re-search-forward hdng-re nil t)
               (-when-let* ((key   (match-string 1))
                            (value (h-get exmp-ht (h-as-symbol key))))
                 (end-of-line)
                 (insert "\n\n" value "\n")))
          (outline-next-heading))
        (goto-char (point-min))
        (outline-hide-subtree)
        (outline-show-branches)))))


;;;;; Show stats

(democratize--describe
  "Commands to show statistics about democratized libraries.")

(defun democratize--calc-ratio (x y)
  "Return ratio between X and Y with one decimal place."
  (if (> y 0)
      (->> (/ x y 1.0)
           (format "%.1f")
           h-as-number)
    "N/A"))

(defun democratize--stats-ht ()
  "Output hash table with stats about democratized libraries."
  (let* ((dems democratize--able-libraries)
         (extr democratize--extracted-dir)
         (libs (h-new (length dems)))
         file  demos  funcs  ratio)
    (dolist (lib dems)
      (setq file (f-expand (format "%s.org" lib) extr))
      (when (f-exists? file)
        (setq demos (democratize--count-examples-in-file file)
              funcs (democratize--count-symbols-in-file  file)
              ratio (democratize--calc-ratio demos funcs))
        (h-put! libs lib (h* "     " lib
                             "demos" demos
                             "funcs" funcs
                             "ratio" ratio))))
    (setq demos (-> libs (h-2d-col "demos") cdr -sum)
          funcs (-> libs (h-2d-col "funcs") cdr -sum)
          ratio (democratize--calc-ratio demos funcs))
    (h-put libs "Total" (h* "     " "Total"
                            "demos" demos
                            "funcs" funcs
                            "ratio" ratio))))

;;;###autoload
(defun democratize-show-stats ()
  "Show stats about democratized libraries."
  (interactive)
  (let* ((buf "*democratize stats*")
         (str '("Statistics about libraries you've democratized"
                "----------------------------------------------" ""
                "Below we have, for each library, the number of:"
                "- demos (examples) extracted"
                "- functions for which there are examples"
                "- examples per function, on average" ""))
         (htb (democratize--stats-ht))
         (lin (->> htb  h->orgtbl  s-lines))
         (out (if (h-empty? htb)
                  "  No libraries have been democratized yet!\n\n"
                (->> lin
                     ;; Add hline before "Total":
                     (-insert-at (1- (length lin)) (cadr lin))
                     (append str)
                     (--map (format "  %s" it))
                     (s-join "\n")))))
    (with-output-to-temp-buffer buf
      (terpri) (princ out)
      (unless (h-empty? htb)
        (with-current-buffer buf
          (when (fboundp 'orgtbl-mode)
            (orgtbl-mode))
          (prog1 nil (read-only-mode)))))))


;;;;; Interface with helpful and help-fns

(democratize--describe
  "Two kinds of help buffers are supported:

1) Emacs' native, \"normal\" *Help* buffer, which shows up when you
   launch help-fns.el's ‘describe-function’ on a function.

2) Helpful buffers, from the library of the same name, and which
   provide, as its author puts it, \"better *help* buffers\", rich
   in additional data about the function.

Democratize provides integration with both. The latter is recommended.")

;;;;;; Enabling examples in help-like buffers

(democratize--describe
  "Commands to make usage examples show up in both kinds of help buffers.")

;;;###autoload
(defun democratize-find-examples (symbol)
  "Find and show examples of SYMBOL in the Org file that has them.
SYMBOL is a function from a democratized library. If there's one at
point, use it. Otherwise, prompt to select from a list."
  (interactive
   (let* ((sym-here    (symbol-at-point))
          (symbols     (democratize--list-symbols))
          (default-val (and sym-here
                            (memq sym-here symbols)
                            (symbol-name sym-here)))
          (prompt      (if default-val
                           (format "Choose (default: %s): " default-val)
                         "Choose: ")))
     (list (read (completing-read prompt
                                  (mapcar #'symbol-name symbols)
                                  nil t nil nil default-val)))))
  (unless symbol
    (error "Assertion failed: %s" symbol))
  (-when-let* ((files (democratize--list-extracted-files))
               (file  (democratize--search-file-that-has-symbol symbol files))
               (pr    (make-progress-reporter
                       (format "Looking for %s ... " symbol))))
    ;; The use of ‘democratize--search-file-that-has-symbol’ above means that
    ;; the file has already been searched and the symbol found. In that sense,
    ;; the below is doing it again, which may seem inefficient. However, that
    ;; search function is fast (no font lock, no modification hooks), whereas
    ;; the below naturally allows fontification of the whole Org buffer —
    ;; which is what we want when displaying. But this means most of the delay
    ;; will be caused by the below, and that the search itself is fast, and
    ;; the extra time spent re-searching the heading is overall negligible. So
    ;; probably not worth optimizing.
    (find-file file)
    (goto-char (point-min))
    (and (re-search-forward
          (format "^\\* %s$" (regexp-quote (symbol-name symbol))))
         (goto-char (line-beginning-position))
         (read-only-mode t)
         (or (if (fboundp 'org-fold-show-entry)
                 (org-fold-show-entry)
               (with-no-warnings (org-show-entry)))
             (progress-reporter-done pr)))))

(defun democratize-help-find-examples-at-point ()
  "When in a help buffer, find in Org file the examples at point."
  (interactive)
  (let ((offset (- (point) (get-text-property (point) 'start))))
    (and (democratize-find-examples (get-text-property (point) 'symbol))
         ;; Skip heading and an empty line
         (forward-line 2)
         (forward-char offset))))

(defvar democratize-help-keymap
  (let ((map (make-sparse-keymap)))
    (define-key map (kbd "RET") #'democratize-help-find-examples-at-point)
    map)
  "Help keymap.")


;;;;;;; helpful

(democratize--describe
  "Commands that provide integration with Helpful buffers.")

;;;###autoload
(defun democratize-insert-examples-into-helpful-buffer ()
  "Insert into ‘helpful’ buffer examples for the function shown."
  (-when-let* ((src (when (symbolp helpful--sym)
                      (democratize--search-symbol-in-extracted-files
                       helpful--sym))))
    (save-excursion
      (goto-char (point-min))
      (when (re-search-forward "^References$")
        (goto-char (line-beginning-position))
        (let* ((inhibit-read-only t)
               (heading (helpful--heading democratize-title-for-helpful))
               (source  (propertize (democratize--syntax-highlight src)
                                    'start (point)
                                    'symbol helpful--sym
                                    'keymap democratize-help-keymap)))
          (insert heading source "\n\n"))))))

;;;###autoload
(defun democratize-enable-examples-in-helpful ()
  "Enable insertion of examples into ‘helpful’ buffer.
Add this to your init file to have this integration enabled when
you start Emacs."
  (interactive)
  (advice-add 'helpful-update :after
              #'democratize-insert-examples-into-helpful-buffer))

;;;###autoload
(defun democratize-find-examples-for-helpful-buffer ()
  "Find in this ‘helpful’ buffer examples for the function shown.
Show these examples in the Org file that has them."
  (interactive)
  (democratize-find-examples helpful--sym))


;;;;;;; help-fns

(democratize--describe
  "Commands that provide integration with native Help buffers.")

;;;###autoload
(defun democratize-insert-examples-into-help-buffer (function)
  "Insert into help buffer examples for the FUNCTION shown.
The help buffer would have been created by ‘describe-function’."
  (-when-let* ((src (when (symbolp function)
                      (democratize--search-symbol-in-extracted-files
                       function)))
               (buf (get-buffer "*Help*")))
    (with-current-buffer buf
      (save-excursion
        (let ((inhibit-read-only t)
              (source (propertize (democratize--syntax-highlight src)
                                  'start (point)
                                  'symbol function
                                  'keymap democratize-help-keymap)))
          (goto-char (point-max))
          (if (not (eq (char-before) ?\n))
              (insert "\n\n")
            (when (not (eq (char-before (1- (point))) ?\n))
              (insert "\n")))
          (insert source "\n")
          (unless (eobp) (insert "\n")))))))

;;;###autoload
(defun democratize-enable-examples-in-help ()
  "Enable insertion of examples into help buffer.
The help buffer would have been created by ‘describe-function’.

Add this to your init file to have this integration enabled when
you start Emacs."
  (interactive)
  (advice-add 'describe-function-1 :after
              #'democratize-insert-examples-into-help-buffer))


;;;;;; Selecting a function to show help and examples for

(democratize--describe
  "Commands to launch a help buffer on a democratized function.")

;;;;;;; helpful

;;;###autoload
(defun democratize-show-helpful-for-function ()
  "Select a democratized function to launch ‘helpful-callable’ on."
  (interactive)
  (require 'helpful)
  (helpful-callable
   (intern (completing-read "Select function to look up on helpful: "
                            (democratize--list-symbols)))))

;;;;;;; help-fns

;;;###autoload
(defun democratize-show-help-for-function ()
  "Select a democratized function to launch ‘describe-function’ on."
  (interactive)
  (describe-function
   (intern (completing-read "Select function to describe: "
                            (democratize--list-symbols)))))


;;;;; See README

(democratize--describe
  "A command to quickly open the README.org of this very library.")

;;;###autoload
(defun democratize-see-readme (&optional heading narrow)
  "Open democratize's README.org file.
Search for the file in democratize.el's directory.

If found, open it read-only.

If optional argument HEADING is passed, try to navigate to the
heading after opening it. HEADING should be a string.

If optional argument NARROW is non-nil, narrow to that heading.
This argument has no effect if HEADING is nil or not found."
  (interactive)
  (let ((readme democratize--readme-org))
    (if (file-exists-p readme)
        (let ((pr (make-progress-reporter
                   (format "Opening %s ... "
                           (abbreviate-file-name readme)))))
          (find-file-read-only readme)
          (when heading
            (democratize--goto-org-heading heading narrow))
          (progress-reporter-done pr))
      (message "Couldn't find %s's README.org" democratize--name))))

(defun democratize--goto-org-heading (heading &optional narrow)
  "Navigate to org HEADING and optionally NARROW to it."
  (let* ((hrx (format "^[*]+ %s" heading))
         (pos (save-match-data
                (save-excursion
                  (save-restriction
                    (widen)
                    (goto-char (point-max))
                    (re-search-backward hrx nil t 1))))))
    (when pos
      (widen)
      (goto-char pos)
      (if (and narrow (fboundp 'org-narrow-to-subtree))
          (org-narrow-to-subtree)
        (recenter-top-bottom 1))
      (when (fboundp 'outline-show-subtree)
        (outline-show-subtree))
      (when (fboundp 'org-flag-drawer)
        (save-excursion
          (forward-line 1)
          (org-flag-drawer t))))))


;;;;; See News

(democratize--describe
  "A command to quickly see News in the README.org.")

;;;###autoload
(defun democratize-see-news ()
  "See the News in democratize's README.org file."
  (interactive)
  (democratize-see-readme "News" 'narrow)
  (democratize--display-org-subtree))

(defun democratize--display-org-subtree ()
  "Selectively display org subtree."
  (let ((cmds '(outline-hide-subtree
                outline-show-children
                outline-next-heading
                outline-show-branches)))
    (and (fboundp (nth 0 cmds))
         (fboundp (nth 1 cmds))
         (fboundp (nth 2 cmds))
         (fboundp (nth 3 cmds))
         (mapc #'funcall cmds))))


;;;; Wrapping up

(provide 'democratize)

;; Local Variables:
;; coding:                     utf-8
;; indent-tabs-mode:           nil
;; sentence-end-double-space:  nil
;; outline-regexp:             ";;;;* "
;; End:

;;; democratize.el ends here