Contents and Indexes

To produce a general index, for HTML output only.

§1. Indexing notations. These allow markup such as this is ^{nifty} to mark headwords in the source documentation for indexing.

Only one of the three ways to add indexing notation actually create a new notation as such: the other two instead piggyback on built-in, i.e., already defined, ones. But in all cases a new "category" is made, corresponding roughly to a CSS class used in the final output.

void Indexes::add_indexing_notation(compiled_documentation *cd, text_stream *L, text_stream *R, text_stream *style, text_stream *options) {
    IndexMarkupNotations::add(cd, L, R, style);
    Indexes::add_category(cd, style, options, NULL);
}

void Indexes::add_indexing_notation_for_definitions(compiled_documentation *cd, text_stream *style, text_stream *options, text_stream *subdef) {
    TEMPORARY_TEXT(key)
    WRITE_TO(key, "!%S", subdef);
    if (Str::len(subdef) > 0) WRITE_TO(key, "-");
    WRITE_TO(key, "definition");
    Indexes::add_category(cd, style, options, key);
    DISCARD_TEXT(key)
}

void Indexes::add_indexing_notation_for_examples(compiled_documentation *cd, text_stream *style, text_stream *options) {
    Indexes::add_category(cd, style, options, I"!example");
}

typedef struct cd_indexing_data {
    int present_with_index;
    struct linked_list *notations;  of index_markup_notation
    struct dictionary *categories_by_name;  to indexing_category
    struct dictionary *categories_redirect;  to text
    struct dictionary *lemmas;  to index_lemma
    struct linked_list *lemma_list;  of index_lemma
} cd_indexing_data;

cd_indexing_data Indexes::new_indexing_data(void) {
    cd_indexing_data id;
    id.present_with_index = FALSE;
    id.notations = NEW_LINKED_LIST(index_markup_notation);
    id.categories_by_name = Dictionaries::new(25, FALSE);
    id.categories_redirect = Dictionaries::new(25, TRUE);
    id.lemmas = Dictionaries::new(100, FALSE);
    id.lemma_list = NEW_LINKED_LIST(index_lemma);
    return id;
}

int Indexes::indexing_occurred(compiled_documentation *cd) {
    return cd->id.present_with_index;
}

The structure cd_indexing_data is accessed in 7/imn and here.

§2. Categories. Categories can be looked up by name (which correspond to CSS class name), and turn out to have a lot of fiddly options added.

typedef struct indexing_category {
    struct text_stream *cat_name;
    struct text_stream *cat_glossed;  if set, print the style as a gloss
    int cat_inverted;  if set, apply name inversion
    struct text_stream *cat_prefix;  if set, prefix to entries
    struct text_stream *cat_suffix;  if set, suffix to entries
    int cat_bracketed;  if set, apply style to bracketed matter
    int cat_unbracketed;  if set, also prune brackets
    int cat_usage;  for counting headwords
    struct text_stream *cat_under;  for automatic subentries
    int cat_alsounder;  for automatic subentries
    CLASS_DEFINITION
} indexing_category;

dictionary *categories_by_name = NULL;
dictionary *categories_redirect = NULL;  for the built-in categories only

The structure indexing_category is private to this section.

§3. Every new style goes into the name dictionary:

void Indexes::add_category(compiled_documentation *cd, text_stream *name, text_stream *options, text_stream *redirect) {
    name = Str::duplicate(name);
    options = Str::duplicate(options);
    redirect = Str::duplicate(redirect);
    if (Str::len(redirect) > 0) This is a redirection3.1;
    if (Dictionaries::find(cd->id.categories_by_name, name) == NULL) {
        indexing_category *ic = CREATE(indexing_category);
        Dictionaries::create(cd->id.categories_by_name, name);
        Dictionaries::write_value(cd->id.categories_by_name, name, ic);
        Work out the fiddly details3.2;
    }
}

§3.1. When we want to say "use my new category X instead of the built-in category Y", we use the redirection dictionary. Here redirect is Y, and name is X.

This is a redirection3.1 =

    text_stream *val = Dictionaries::create_text(cd->id.categories_redirect, redirect);
    Str::copy(val, name);

This code is used in §3.

§3.2. There's a whole little mini-language for how to express details of our category:

Work out the fiddly details3.2 =

    ic->cat_name = Str::duplicate(name);
    match_results mr = Regexp::create_mr();
    ic->cat_glossed = Str::new();
    if (Regexp::match(&mr, options, U"(%c*?) *%(\"(%c*?)\"%) *(%c*)")) {
        ic->cat_glossed = Str::duplicate(mr.exp[1]);
        Str::clear(options); WRITE_TO(options, "%S%S", mr.exp[0], mr.exp[2]);
    }
    ic->cat_prefix = Str::new();
    if (Regexp::match(&mr, options, U"(%c*?) *%(prefix \"(%c*?)\"%) *(%c*)")) {
        ic->cat_prefix = Str::duplicate(mr.exp[1]);
        Str::clear(options); WRITE_TO(options, "%S%S", mr.exp[0], mr.exp[2]);
    }
    ic->cat_suffix = Str::new();
    if (Regexp::match(&mr, options, U"(%c*?) *%(suffix \"(%c*?)\"%) *(%c*)")) {
        ic->cat_suffix = Str::duplicate(mr.exp[1]);
        Str::clear(options); WRITE_TO(options, "%S%S", mr.exp[0], mr.exp[2]);
    }
    ic->cat_under = Str::new();
    if (Regexp::match(&mr, options, U"(%c*?) *%(under {(%c*?)}%) *(%c*)")) {
        Str::clear(options); WRITE_TO(options, "%S%S", mr.exp[0], mr.exp[2]);
        ic->cat_under = Str::duplicate(mr.exp[1]);
    }
    ic->cat_alsounder = FALSE;
    if (Regexp::match(&mr, options, U"(%c*?) *%(also under {(%c*?)}%) *(%c*)")) {
        Str::clear(options); WRITE_TO(options, "%S%S", mr.exp[0], mr.exp[2]);
        ic->cat_under = Str::duplicate(mr.exp[1]);
        ic->cat_alsounder = TRUE;
    }
    ic->cat_inverted = FALSE;
    if (Regexp::match(&mr, options, U"(%c*?) *%(invert%) *(%c*)")) {
        ic->cat_inverted = TRUE;
        Str::clear(options); WRITE_TO(options, "%S%S", mr.exp[0], mr.exp[1]);
    }
    ic->cat_bracketed = FALSE;
    if (Regexp::match(&mr, options, U"(%c*?) *%(bracketed%) *(%c*)")) {
        ic->cat_bracketed = TRUE;
        Str::clear(options); WRITE_TO(options, "%S%S", mr.exp[0], mr.exp[1]);
    }
    ic->cat_unbracketed = FALSE;
    if (Regexp::match(&mr, options, U"(%c*?) *%(unbracketed%) *(%c*)")) {
        ic->cat_bracketed = TRUE;
        ic->cat_unbracketed = TRUE;
        Str::clear(options); WRITE_TO(options, "%S%S", mr.exp[0], mr.exp[1]);
    }
    if (Regexp::match(NULL, options, U"%c*?%C%c*"))
        Errors::with_text("Unknown notation options: %S", options);
    ic->cat_usage = 0;
    Regexp::dispose_of(&mr);

This code is used in §3.

§4. The following looks slow, but in fact there's no problem in practice.

void Indexes::scan(compiled_documentation *cd) {
    markdown_item *latest = cd->markdown_content;
    int volume_number = -1;
    Indexes::scan_r(cd, cd->markdown_content, &latest, NULL, &volume_number);
    volume_number = -1;
    IFM_example *E;
    LOOP_OVER_LINKED_LIST(E, IFM_example, cd->examples)
        Indexes::scan_r(cd, E->header, NULL, E, &volume_number);
    if (LinkedLists::len(cd->id.lemma_list) > LinkedLists::len(cd->examples))
        cd->id.present_with_index = TRUE;
}

void Indexes::scan_r(compiled_documentation *cd, markdown_item *md, markdown_item **latest,
    IFM_example *E, int *volume_number) {
    if (md) {
        if (md->type == VOLUME_MIT) (*volume_number)++;
        if ((md->type == HEADING_MIT) && (Markdown::get_heading_level(md) <= 2)) {
            if (latest) *latest = md;
        }
        if (md->type == INDEX_MARKER_MIT) {
            Indexes::scan_indexingnotations(cd, md, md->details, md->stashed, *volume_number,
                (latest)?(*latest):NULL, E);
        }
        if ((md->type == INFORM_EXAMPLE_HEADING_MIT) && (latest) && (*latest)) {
            IFM_example *EG = RETRIEVE_POINTER_IFM_example(md->user_state);
            TEMPORARY_TEXT(term)
            Indexes::extract_from_indexable_matter(term, cd, Str::duplicate(EG->name), NULL);
            Indexes::mark_index_term(cd, term, *volume_number, *latest, NULL, EG, NULL, NULL, 1);
            if ((Str::len(EG->ex_index) > 0) && (Str::ne(EG->ex_index, EG->name))) {
                Str::clear(term);
                Indexes::extract_from_indexable_matter(term, cd, Str::duplicate(EG->ex_index), NULL);
                Indexes::mark_index_term(cd, term, *volume_number, *latest, NULL, EG, NULL, NULL, 2);
            }
            DISCARD_TEXT(term)
            E = EG;
        }
    }
    for (markdown_item *ch = md->down; ch; ch=ch->next)
        Indexes::scan_r(cd, ch, latest, E, volume_number);
}

void Indexes::scan_indexingnotations(compiled_documentation *cd, markdown_item *md,
    int carets, text_stream *term_to_index, int V, markdown_item *S, IFM_example *E) {
    match_results mr = Regexp::create_mr();
    TEMPORARY_TEXT(see)
    TEMPORARY_TEXT(alphabetise_as)
    if (Regexp::match(&mr, term_to_index, U"(%c+?) *<-- *(%c+) *")) {
        Str::copy(term_to_index, mr.exp[0]); Str::copy(see, mr.exp[1]);
    }
    if (Regexp::match(&mr, term_to_index, U"(%c+?) *--> *(%c+) *")) {
        Str::copy(term_to_index, mr.exp[0]); Str::copy(alphabetise_as, mr.exp[1]);
    }
    TEMPORARY_TEXT(lemma)
    Indexes::extract_from_indexable_matter(lemma, cd, term_to_index, (carets==1)?(md->next):NULL);

    if ((V > 0) && (E)) {
        V = 0; S = NULL; E = NULL;
    }
    if (carets < 3) {
        Indexes::mark_index_term(cd, lemma, V, S, NULL, E, NULL, alphabetise_as, FALSE);
    } else {
        Indexes::note_index_term_alphabetisation(cd, lemma, alphabetise_as);
    }

    TEMPORARY_TEXT(smoke_test_text)
    Indexes::process_category_options(smoke_test_text, cd, lemma, TRUE, 1);

    while (Regexp::match(&mr, see, U" *(%c+) *<-- *(%c+?) *")) {
        Str::copy(see, mr.exp[0]);
        TEMPORARY_TEXT(seethis)
        Indexes::extract_from_indexable_matter(seethis, cd, mr.exp[1], NULL);
        Indexes::mark_index_term(cd, seethis, -1, NULL, NULL, NULL, lemma, NULL, FALSE);
        WRITE_TO(smoke_test_text, " <-- ");
        Indexes::process_category_options(smoke_test_text, cd, seethis, TRUE, 2);
        DISCARD_TEXT(seethis)
    }
    if (Str::len(see) > 0) {
        TEMPORARY_TEXT(seethis)
        Indexes::extract_from_indexable_matter(seethis, cd, see, NULL);
        Indexes::mark_index_term(cd, seethis, -1, NULL, NULL, NULL, lemma, NULL, FALSE);
        WRITE_TO(smoke_test_text, " <-- ");
        Indexes::process_category_options(smoke_test_text, cd, seethis, TRUE, 3);
        DISCARD_TEXT(seethis)
    }
    if (indoc_settings_test_index_mode) {
        Regexp::replace(smoke_test_text, U"=___=standard", U"", REP_REPEATING);
        Regexp::replace(smoke_test_text, U"=___=(%C+)", U" %(%0%)", REP_REPEATING);
        Regexp::replace(smoke_test_text, U":", U": ", REP_REPEATING);
        md->type = PLAIN_MIT;
        md->sliced_from = Str::duplicate(smoke_test_text);
        md->from = 0; md->to = Str::len(smoke_test_text) - 1;
    }
    DISCARD_TEXT(lemma)
    DISCARD_TEXT(see)
    DISCARD_TEXT(alphabetise_as)
    DISCARD_TEXT(smoke_test_text)
    Regexp::dispose_of(&mr);
}

void Indexes::extract_from_indexable_matter(OUTPUT_STREAM, compiled_documentation *cd,
    text_stream *text, markdown_item *plain_md) {
    match_results mr = Regexp::create_mr();
    if (Regexp::match(&mr, text, U" *(%c+?) *: *(%c+) *")) {
        text_stream *head = mr.exp[0];
        text_stream *tail = mr.exp[1];
        Indexes::extract_from_indexable_matter(OUT, cd, head, NULL);
        WRITE(":");
        Indexes::extract_from_indexable_matter(OUT, cd, tail, NULL);
        Regexp::dispose_of(&mr);
        return;
    }
    Regexp::dispose_of(&mr);
    TEMPORARY_TEXT(trimmed)
    Str::copy(trimmed, text);
    Str::trim_white_space(trimmed);
    index_markup_notation *imn = IndexMarkupNotations::match(cd, trimmed);
    if (imn) {
        int L = IndexMarkupNotations::left_width(imn), R = IndexMarkupNotations::right_width(imn);
        for (int j=L; j < Str::len(text) - R; j++) PUT(Str::get_at(text, j));
        WRITE("=___=%S", IndexMarkupNotations::style_name(imn));
        if ((plain_md) && (plain_md->type == PLAIN_MIT)) {
            plain_md->from += L;
            plain_md->to -= R;
        }
    } else {
        WRITE("%S=___=standard", trimmed);
    }
    DISCARD_TEXT(trimmed)
}

§5.

void Indexes::mark_index_term(compiled_documentation *cd, text_stream *given_term, int V, markdown_item *S,
    text_stream *anchor, IFM_example *E, text_stream *see, text_stream *alphabetise_as,
    int example_index_status) {
    TEMPORARY_TEXT(term)
    Indexes::process_category_options(term, cd, given_term, TRUE, 4);
    if ((Regexp::match(NULL, term, U"IGNORE=___=ME%c*")) ||
        (Regexp::match(NULL, term, U"%c*:IGNORE=___=ME%c*"))) return;
    if (Str::len(alphabetise_as) > 0)
        IndexUtilities::alphabetisation_exception(term, alphabetise_as);
    Indexes::ensure_lemmas_exist(cd, term, example_index_status);
    match_results mr = Regexp::create_mr();
    if (Regexp::match(&mr, term, U"%c*=___=([^_]+?)")) {
        text_stream *category = mr.exp[0];
        if (Dictionaries::find(cd->id.categories_by_name, category)) {
            indexing_category *ic = (indexing_category *)
                Dictionaries::read_value(cd->id.categories_by_name, category);
            Regexp::dispose_of(&mr);
            if ((ic) && (ic->cat_alsounder == TRUE)) {
                TEMPORARY_TEXT(processed_term)
                Indexes::process_category_options(processed_term, cd, given_term, FALSE, 5);
                if ((Regexp::match(NULL, processed_term, U"IGNORE=___=ME%c*")) ||
                    (Regexp::match(NULL, processed_term, U"%c*:IGNORE=___=ME%c*"))) return;
                Indexes::ensure_lemmas_exist(cd, processed_term, example_index_status);
                Indexes::set_index_point(cd, processed_term, V, S, anchor, E, see,
                    example_index_status);
                DISCARD_TEXT(processed_term)
            }
        }
    }
    Indexes::set_index_point(cd, term, V, S, anchor, E, see, example_index_status);
    DISCARD_TEXT(term)
}

§6.

typedef struct index_lemma {
    struct text_stream *term;  text of lemma
    struct linked_list *index_points;  of index_reference
    struct text_stream *index_see;  <---separated list of refs
    struct text_stream *sorting_key;  final reading order is alphabetic on this
    int example_index_status;  as well as in the general index
    CLASS_DEFINITION
} index_lemma;

typedef struct index_reference {
    int volume;
    struct IFM_example *example;
    struct markdown_item *section;
    struct text_stream *anchor;
    CLASS_DEFINITION
} index_reference;

void Indexes::ensure_lemmas_exist(compiled_documentation *cd, text_stream *text,
    int example_index_status) {
    match_results mr = Regexp::create_mr();
    if (Regexp::match(&mr, text, U" *(%c+) *: *(%c+?) *"))
        Indexes::ensure_lemmas_exist(cd, mr.exp[0], example_index_status);
    Regexp::dispose_of(&mr);
    if (Dictionaries::find(cd->id.lemmas, text) == NULL) {
        TEMPORARY_TEXT(copied)
        Str::copy(copied, text);
        Indexes::set_index_point(cd, copied, -1, NULL, NULL, NULL, NULL, example_index_status);
        DISCARD_TEXT(copied)
    }
}

void Indexes::set_index_point(compiled_documentation *cd, text_stream *term, int V, markdown_item *S,
    text_stream *anchor, IFM_example *E, text_stream *see, int example_index_status) {
    index_lemma *il = NULL;
    if (Dictionaries::find(cd->id.lemmas, term)) {
        il = (index_lemma *) Dictionaries::read_value(cd->id.lemmas, term);
    } else {
        Dictionaries::create(cd->id.lemmas, term);
        il = CREATE(index_lemma);
        il->term = Str::duplicate(term);
        il->index_points = NEW_LINKED_LIST(index_reference);
        il->index_see = Str::new();
        il->sorting_key = Str::new();
        il->example_index_status = example_index_status;
        Dictionaries::write_value(cd->id.lemmas, term, il);
        ADD_TO_LINKED_LIST(il, index_lemma, cd->id.lemma_list);
    }
    if ((V >= 0) && ((E) || (S))) {
        index_reference *ref = CREATE(index_reference);
        ref->volume = V;
        ref->example = E;
        ref->section = S;
        ref->anchor = Str::duplicate(anchor);
        ADD_TO_LINKED_LIST(ref, index_reference, il->index_points);
    }
    if (Str::len(see) > 0) WRITE_TO(il->index_see, "%S<--", see);
}

The structure index_lemma is private to this section.
The structure index_reference is private to this section.

§7.

void Indexes::note_index_term_alphabetisation(compiled_documentation *cd,
    text_stream *term, text_stream *alphabetise_as) {
    TEMPORARY_TEXT(processed_term)
    Indexes::process_category_options(processed_term, cd, term, TRUE, 6);
    IndexUtilities::alphabetisation_exception(processed_term, alphabetise_as);
    DISCARD_TEXT(processed_term)
}

void Indexes::process_category_options(OUTPUT_STREAM, compiled_documentation *cd,
    text_stream *text, int allow_under, int n) {
    match_results mr = Regexp::create_mr();
    Break the text down into a colon-separated list of categories and process each7.1;
    if (Regexp::match(&mr, text, U"(%c*)=___=(%c*)")) {
        text_stream *lemma = mr.exp[0];
        text_stream *category = mr.exp[1];
        Redirect category names starting with an exclamation7.2;
        Amend the lemma or category as necessary7.3;
        WRITE("%S=___=%S", lemma, category);
    } else {
        Errors::with_text("bad indexing term: %S", text);
        WRITE("IGNORE=___=ME");
    }
    Regexp::dispose_of(&mr);
}

§7.1. Break the text down into a colon-separated list of categories and process each7.1 =

    if (Regexp::match(&mr, text, U" *(%c+?) *: *(%c+)")) {
        Indexes::process_category_options(OUT, cd, mr.exp[0], TRUE, 7);
        WRITE(":");
        Indexes::process_category_options(OUT, cd, mr.exp[1], allow_under, 8);
        Regexp::dispose_of(&mr);
        return;
    }

This code is used in §7.

§7.2. A category beginning ! is either redirected to a regular category, or else suppressed as unwanted (because the user didn't set up a redirection).

Redirect category names starting with an exclamation7.2 =

    if (Str::get_first_char(category) == '!') {
        text_stream *redirected =
            Dictionaries::get_text(cd->id.categories_redirect, category);
        if (Str::len(redirected) > 0) Str::copy(category, redirected);
        else {
            Regexp::dispose_of(&mr);
            WRITE("IGNORE=___=ME");
            return;
        }
    }

This code is used in §7.

§7.3. Amend the lemma or category as necessary7.3 =

    if (Dictionaries::find(cd->id.categories_by_name, category)) {
        indexing_category *ic = (indexing_category *)
            Dictionaries::read_value(cd->id.categories_by_name, category);
        if (ic) {
            Perform name inversion as necessary7.3.1;
            Prefix and suffix as necessary7.3.2;
            Automatically file under a headword as necessary7.3.3;
        }
    }

This code is used in §7.

§7.3.1. This inverts "Sir Robert Cecil" to "Cecil, Sir Robert", but leaves "Mary, Queen of Scots" alone.

Perform name inversion as necessary7.3.1 =

    if ((ic->cat_inverted) && (Regexp::match(NULL, lemma, U"%c*,%c*") == FALSE)) {
        match_results mr = Regexp::create_mr();
        if (Regexp::match(&mr, lemma, U"(%c*?) (%C+) *")) {
            Str::clear(lemma);
            WRITE_TO(lemma, "%S, %S", mr.exp[1], mr.exp[0]);
        }
        Regexp::dispose_of(&mr);
    }

This code is used in §7.3.

§7.3.2. This, for example, could append "(monarch)" to the name of every lemma in the category "royalty", so that "James I" becomes "James I (monarch)".

Prefix and suffix as necessary7.3.2 =

    TEMPORARY_TEXT(rewritten)
    WRITE_TO(rewritten, "%S%S%S", ic->cat_prefix, lemma, ic->cat_suffix);
    Str::copy(lemma, rewritten);
    DISCARD_TEXT(rewritten)

This code is used in §7.3.

§7.3.3. And this could automatically reroute the lemma so that it appears as a subentry under the category's choice of headword: e.g., "James I" might be placed as as a subentry of "Kings".

Automatically file under a headword as necessary7.3.3 =

    if ((allow_under) && (Str::len(ic->cat_under) > 0)) {
        TEMPORARY_TEXT(extracted)
        TEMPORARY_TEXT(icu)
        TEMPORARY_TEXT(old_lemma)
        Str::copy(old_lemma, lemma);

        Indexes::extract_from_indexable_matter(extracted, cd, ic->cat_under, NULL);
        Indexes::process_category_options(icu, cd, extracted, FALSE, 9);
        Str::clear(lemma);
        WRITE_TO(lemma, "%S:%S", icu, old_lemma);

        DISCARD_TEXT(extracted)
        DISCARD_TEXT(old_lemma)
        DISCARD_TEXT(icu)
    }

This code is used in §7.3.

§8. Rendering. Having accumulated the lemmas, it's time to sort them and write the index as it will be seen by the reader.

void Indexes::write_example_index(OUTPUT_STREAM, compiled_documentation *cd) {
    Indexes::write_general_index_inner(OUT, cd, TRUE);
}

void Indexes::write_general_index(OUTPUT_STREAM, compiled_documentation *cd) {
    Indexes::write_general_index_inner(OUT, cd, FALSE);
}

void Indexes::write_general_index_inner(OUTPUT_STREAM, compiled_documentation *cd,
    int just_examples) {
    HTML_OPEN_WITH("div", "class=\"generalindex\"");
    Construct sorting keys for the lemmas8.1;
    int NL = LinkedLists::len(cd->id.lemma_list);
    index_lemma **lemma_list =
        Memory::calloc(NL, sizeof(index_lemma *), ARRAY_SORTING_MREASON);
    index_lemma *il; int i=0;
    LOOP_OVER_LINKED_LIST(il, index_lemma, cd->id.lemma_list) lemma_list[i++] = il;
    qsort(lemma_list, (size_t) NL, sizeof(index_lemma *), Indexes::sort_comparison);
    Render the index in sorted order8.2;
    Give feedback in index testing mode8.3;
    Memory::I7_free(lemma_list, ARRAY_SORTING_MREASON, NL*((int) sizeof(index_lemma *)));
    HTML_CLOSE("div");
}

int Indexes::sort_comparison(const void *ent1, const void *ent2) {
    const index_lemma *L1 = *((const index_lemma **) ent1);
    const index_lemma *L2 = *((const index_lemma **) ent2);
    return Str::cmp(L1->sorting_key, L2->sorting_key);
}

§8.1. Construct sorting keys for the lemmas8.1 =

    index_lemma *il;
    LOOP_OVER_LINKED_LIST(il, index_lemma, cd->id.lemma_list) {
        TEMPORARY_TEXT(sort_key)
        Str::copy(sort_key, il->term);
         ensure subentries follow main entries
        if (Str::get_first_char(sort_key) != ':')
            Regexp::replace(sort_key, U": *", U"ZZZZZZZZZZZZZZZZZZZZZZ", REP_REPEATING);
        IndexUtilities::improve_alphabetisation(sort_key);

        match_results mr = Regexp::create_mr();
        if (Regexp::match(&mr, sort_key, U"a/%C+ (%c*)")) Str::copy(sort_key, mr.exp[0]);
        if (Regexp::match(&mr, sort_key, U"the/%C+ (%c*)")) Str::copy(sort_key, mr.exp[0]);

        if (indoc_settings_index_alphabetisation_algorithm == WORD_ALPHABETIZATION)
            Regexp::replace(sort_key, U" ", U"aaaaaaaaaaaaaaaaaaaaaa", REP_REPEATING);

        TEMPORARY_TEXT(un)
        Str::copy(un, sort_key);
        if ((Str::begins_with(sort_key, I"( )") == FALSE) &&
            (Str::begins_with(sort_key, I"((-") == FALSE) &&
            (Str::begins_with(sort_key, I"((+") == FALSE))
            Regexp::replace(un, U"%(%c*?%)", NULL, REP_REPEATING);
        Regexp::replace(un, U" ", NULL, REP_REPEATING);
        if (Str::get_first_char(sort_key) != ',')
            Regexp::replace(un, U",", NULL, REP_REPEATING);
        inchar32_t f = ' ';
        if (Characters::isalpha(Str::get_first_char(sort_key)))
            f = Str::get_first_char(sort_key);
        WRITE_TO(il->sorting_key, "%c_%S=___=%S=___=%07d",
            f, un, sort_key, il->allocation_id);

        DISCARD_TEXT(un)
        DISCARD_TEXT(sort_key)
        Regexp::dispose_of(&mr);
    }

This code is used in §8.

§8.2. Render the index in sorted order8.2 =

    IndexUtilities::alphabet_row(OUT, 1);
    HTML_OPEN_WITH("table", "class=\"indextable\"");
    inchar32_t current_incipit = 0;
    for (int i=0; i<NL; i++) {
        index_lemma *il = lemma_list[i];
        if ((just_examples) && (il->example_index_status == 0)) continue;
        if ((!just_examples) && (il->example_index_status == 2)) continue;
        inchar32_t incipit = Str::get_first_char(il->sorting_key);
        if (Characters::isalpha(incipit)) incipit = Characters::toupper(incipit);
        else incipit = '#';
        if (incipit != current_incipit) {
            if (current_incipit != 0) End a block of the index8.2.2;
            current_incipit = incipit;
            IndexUtilities::note_letter(current_incipit);
            Start a block of the index8.2.1;
        }
        Render an index entry8.2.3;
    }
    if (current_incipit != 0) End a block of the index8.2.2;
    HTML_CLOSE("table");
    IndexUtilities::alphabet_row(OUT, 2);

This code is used in §8.

§8.2.1. Start a block of the index8.2.1 =

    HTML_OPEN("tr");
    HTML_OPEN_WITH("td", "class=\"letterblock\"");
    TEMPORARY_TEXT(inc)
    if (current_incipit == '#') WRITE_TO(inc, "NN");
    else PUT_TO(inc, current_incipit);
    HTML::anchor(OUT, inc);
    IndexUtilities::majuscule_heading(OUT, inc, TRUE);
    DISCARD_TEXT(inc)
    HTML_CLOSE("td");
    HTML_OPEN("td");

This code is used in §8.2.

§8.2.2. End a block of the index8.2.2 =

    HTML_CLOSE("td");
    HTML_CLOSE("tr");

This code is used in §8.2 (twice).

§8.2.3. Render an index entry8.2.3 =

    TEMPORARY_TEXT(anc)
    int A = il->allocation_id;
    WRITE_TO(anc, "l%d", A);
    HTML::anchor(OUT, anc);
    DISCARD_TEXT(anc)

    TEMPORARY_TEXT(term)
    TEMPORARY_TEXT(category)
    match_results mr = Regexp::create_mr();
    Str::copy(term, il->term);
    if (Regexp::match(&mr, term, U"(%c*)=___=(%c*)")) {
        Str::copy(term, mr.exp[0]);
        Str::copy(category, mr.exp[1]);
    }
    indexing_category *ic = NULL;
    if (Dictionaries::find(cd->id.categories_by_name, category) == NULL) {
        if (Str::eq_insensitive(category, I"standard") == FALSE)
            PRINT("Warning: no such indexing category as '%S'\n", category);
    } else {
        ic = Dictionaries::read_value(cd->id.categories_by_name, category);
        ic->cat_usage++;

        int indent_level = 0;
        TEMPORARY_TEXT(lemma_wording)
        Work out the wording and indentation level8.2.3.1;

        TEMPORARY_TEXT(details)
        WRITE_TO(details, "class=\"indexentry\" style=\"margin-left: %dem;\"", 4*indent_level);
        HTML::open(OUT, "p", details, __FILE__, __LINE__);
        DISCARD_TEXT(details)
        IFM_example *EG = NULL;
        if (il->example_index_status > 0) {
            index_reference *ref;
            LOOP_OVER_LINKED_LIST(ref, index_reference, il->index_points)
                if (ref->example)
                    EG = ref->example;
        }
        Render the lemma text8.2.3.2;
        Render the category gloss8.2.3.3;
        WRITE("&nbsp;&nbsp;");
        int lc = 0;
        Render the list of index points8.2.3.4;
        Render the list of see-references8.2.3.5;
        HTML_CLOSE("p");
        Regexp::dispose_of(&mr);
    }

This code is used in §8.2.

§8.2.3.1.

define SAVED_OPEN_BRACKET 0x0086   Unicode "start of selected area"
define SAVED_CLOSE_BRACKET 0x0087  Unicode "end of selected area"

Work out the wording and indentation level8.2.3.1 =

    TEMPORARY_TEXT(untreated)
    Str::copy(untreated, term);
    while (Regexp::match(&mr, untreated, U"%c*?: *(%c+)")) {
        Str::copy(untreated, mr.exp[0]); indent_level++;
    }
    IndexUtilities::escape_HTML_characters_in(untreated);
    for (int i=0, L = Str::len(untreated); i<L; i++) {
        inchar32_t c = Str::get_at(untreated, i);
        if (c == '\\') {
            inchar32_t n = Str::get_at(untreated, ++i);
            if (n == '(') n = SAVED_OPEN_BRACKET;
            if (n == ')') n = SAVED_CLOSE_BRACKET;
            PUT_TO(lemma_wording, n);
        } else PUT_TO(lemma_wording, c);
    }

    if (ic->cat_bracketed) {
        while (Regexp::match(&mr, lemma_wording, U"(%c*?)%(%(%+ %+%)%)(%c*)")) {
            Str::clear(lemma_wording);
            WRITE_TO(lemma_wording,
                "%S<span class=\"index%Sbracketed\">%c+ +%c</span>%S",
                mr.exp[0], category, SAVED_OPEN_BRACKET, SAVED_CLOSE_BRACKET, mr.exp[1]);
        }
        while (Regexp::match(&mr, lemma_wording, U"(%c*?)%(%(%- %-%)%)(%c*)")) {
            Str::clear(lemma_wording);
            WRITE_TO(lemma_wording,
                "%S<span class=\"index%Sbracketed\">%c- -%c</span>%S",
                mr.exp[0], category, SAVED_OPEN_BRACKET, SAVED_CLOSE_BRACKET, mr.exp[1]);
        }
        while (Regexp::match(&mr, lemma_wording, U"(%c*?)%((%c*?)%)(%c*)")) {
            Str::clear(lemma_wording);
            WRITE_TO(lemma_wording,
                "%S<span class=\"index%Sbracketed\">___openb___%S___closeb___</span>%S",
                mr.exp[0], category, mr.exp[1], mr.exp[2]);
        }
        if (ic->cat_unbracketed) {
            Regexp::replace(lemma_wording, U"___openb___", NULL, REP_REPEATING);
            Regexp::replace(lemma_wording, U"___closeb___", NULL, REP_REPEATING);
        } else {
            Regexp::replace(lemma_wording, U"___openb___", U"(", REP_REPEATING);
            Regexp::replace(lemma_wording, U"___closeb___", U")", REP_REPEATING);
        }
    }

    LOOP_THROUGH_TEXT(pos, lemma_wording) {
        inchar32_t d = Str::get(pos);
        if (d == SAVED_OPEN_BRACKET) Str::put(pos, '(');
        if (d == SAVED_CLOSE_BRACKET) Str::put(pos, ')');
    }

This code is used in §8.2.3.

§8.2.3.2. Render the lemma text8.2.3.2 =

    if (il->example_index_status == 1) {
        HTML_OPEN("b");
        if (EG) HTML_OPEN_WITH("a", "href=\"%S\"", EG->URL);
    }
    HTML_OPEN_WITH("span", "class=\"index%S\"", category);
    WRITE("%S", lemma_wording);
    HTML_CLOSE("span");
    if (il->example_index_status == 1) {
        if (EG) HTML_CLOSE("a");
        HTML_CLOSE("b");
    }

This code is used in §8.2.3.

§8.2.3.3. Render the category gloss8.2.3.3 =

    if (Str::len(ic->cat_glossed) > 0)
        WRITE("&nbsp;<span class=\"indexgloss\">%S</span>", ic->cat_glossed);

This code is used in §8.2.3.

§8.2.3.4. Render the list of index points8.2.3.4 =

    index_reference *ref;
    LOOP_OVER_LINKED_LIST(ref, index_reference, il->index_points) {
        if (lc++ > 0) WRITE(", ");

        int volume_number = ref->volume;
        markdown_item *S = ref->section;

        IFM_example *E = ref->example;
        if ((E) && (S == NULL)) S = E->cue;
        if ((S == NULL) && (E == NULL))
            internal_error("unknown destination in index reference");

        text_stream *link_class = I"indexlink";
        if (volume_number > 0) link_class = I"indexlinkalt";
        TEMPORARY_TEXT(link)
        text_stream *A = ref->anchor;
        if (S) {
            for (int i=0; i<Str::len(S->stashed); i++) {
                inchar32_t c = Str::get_at(S->stashed, i);
                if (c == ':') break;
                if ((Characters::isdigit(c)) || (c == '.')) PUT_TO(link, c);
            }
            A = MarkdownVariations::URL_for_heading(S);
        }
        if (E) {
            if (S) WRITE_TO(link, " ");
            WRITE_TO(link, "ex %S", E->insignia);
            if (EG == NULL) A = E->URL;
        }
        IndexUtilities::general_link(OUT, link_class, A, link);
        DISCARD_TEXT(link)
    }

This code is used in §8.2.3.

§8.2.3.5. Render the list of see-references8.2.3.5 =

    TEMPORARY_TEXT(seelist)
    Str::copy(seelist, il->index_see);
    int sc = 0;
    if (Str::len(seelist) > 0) {
        if (lc > 0) WRITE("; ");
        HTML_OPEN_WITH("span", "class=\"indexsee\"");
        WRITE("see ");
        if (lc > 0) WRITE("also ");
        HTML_CLOSE("span");
        match_results mr2 = Regexp::create_mr();
        while (Regexp::match(&mr2, seelist, U"(%c*?) *<-- *(%c*)")) {
            if (sc++ > 0) { WRITE("; "); }

            text_stream *see = mr2.exp[0];
            Str::copy(seelist, mr2.exp[1]);
            index_lemma *ils = (index_lemma *) Dictionaries::read_value(cd->id.lemmas, see);
            TEMPORARY_TEXT(url)
            WRITE_TO(url, "#l%d", ils->allocation_id);
            Regexp::replace(see, U"=___=%i+?:", U":", REP_REPEATING);
            Regexp::replace(see, U"=___=%i+", NULL, REP_REPEATING);
            Regexp::replace(see, U":", U": ", REP_REPEATING);
            IndexUtilities::general_link(OUT, I"indexseelink", url, see);
            DISCARD_TEXT(url)
        }
        Regexp::dispose_of(&mr2);
    }

This code is used in §8.2.3.

§8.3. Give feedback in index testing mode8.3 =

    if (indoc_settings_test_index_mode) {
        PRINT("indoc ran in index test mode: do not publish typeset documentation.\n");
        int t = 0;
        indexing_category *ic;
        LOOP_OVER(ic, indexing_category) {
            PRINT("%S: %d headword(s)\n", ic->cat_name, ic->cat_usage);
            t += ic->cat_usage;
        }
        PRINT("%d headword(s) in all\n", t);
    }

This code is used in §8.

§9.

void Indexes::render_eg_index(OUTPUT_STREAM, markdown_item *md) {
    markdown_item *pending_chapter = NULL, *pending_section = NULL;
    if (md) Indexes::render_eg_index_r(OUT, md, &pending_chapter, &pending_section);
}

void Indexes::render_eg_index_r(OUTPUT_STREAM, markdown_item *md,
    markdown_item **pending_chapter, markdown_item **pending_section) {
    if ((md->type == HEADING_MIT) && (Markdown::get_heading_level(md) == 1)) {
        *pending_chapter = md;
        *pending_section = NULL;
    }
    if ((md->type == HEADING_MIT) && (Markdown::get_heading_level(md) == 2)) {
        *pending_section = md;
    }
    if (md->type == INFORM_EXAMPLE_HEADING_MIT) {
        if (*pending_chapter) {
            Markdown::render_extended(OUT, *pending_chapter, InformFlavouredMarkdown::variation());
            *pending_chapter = NULL;
        }
        if (*pending_section) {
            Markdown::render_extended(OUT, *pending_section, InformFlavouredMarkdown::variation());
            *pending_section = NULL;
        }
        Markdown::render_extended(OUT, md, InformFlavouredMarkdown::variation());
    }
    for (markdown_item *ch = md->down; ch; ch = ch->next)
        Indexes::render_eg_index_r(OUT, ch, pending_chapter, pending_section);
}