/*************************************************************************/
/* Copyright (c) 2004 */
/* Daniel Sleator, David Temperley, and John Lafferty */
/* Copyright 2008, 2009, 2013, 2014 Linas Vepstas */
/* All rights reserved */
/* */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software. */
/* This license allows free redistribution and use in source and binary */
/* forms, with or without modification, subject to certain conditions. */
/* */
/*************************************************************************/
#include <limits.h>
#include "api-structures.h"
#include "count.h"
#include "dict-common/dict-common.h" // For Dictionary_s
#include "disjunct-utils.h"
#include "extract-links.h"
#include "fast-match.h"
#include "linkage/analyze-linkage.h"
#include "linkage/linkage.h"
#include "linkage/sane.h"
#include "parse.h"
#include "post-process/post-process.h"
#include "preparation.h"
#include "prune.h"
#include "resources.h"
#include "tokenize/word-structures.h" // For Word_struct
#define D_PARSE 5 /* Debug level for this file. */
static Linkage linkage_array_new(int num_to_alloc)
{
Linkage lkgs = (Linkage) malloc(num_to_alloc * sizeof(struct Linkage_s));
memset(lkgs, 0, num_to_alloc * sizeof(struct Linkage_s));
return lkgs;
}
void linkage_array_free(Linkage lkgs)
{
free(lkgs);
}
static void find_unused_disjuncts(Sentence sent, extractor_t *pex)
{
const size_t disjunct_used_sz =
sizeof(bool) * sent->wildcard_word_num_disjuncts;
sent->disjunct_used = malloc(disjunct_used_sz);
memset(sent->disjunct_used, 0, disjunct_used_sz);
if (pex != NULL)
mark_used_disjuncts(pex, sent->disjunct_used);
if (verbosity_level(+D_PARSE))
{
unsigned int num_unused = 0;
for (unsigned int i = 0; i < sent->wildcard_word_num_disjuncts; i++)
if (!sent->disjunct_used[i]) num_unused++;
prt_error("Info: Unused disjuncts %u/%u\n", num_unused,
sent->wildcard_word_num_disjuncts);
}
}
static void setup_linkages(Sentence sent, extractor_t* pex,
fast_matcher_t* mchxt,
count_context_t* ctxt,
Parse_Options opts)
{
sent->overflowed = build_parse_set(pex, sent, mchxt, ctxt, sent->null_count, opts);
print_time(opts, "Built parse set");
if (sent->overflowed && (1 < opts->verbosity) && !IS_GENERATION(sent->dict))
{
err_ctxt ec = { sent };
err_msgc(&ec, lg_Warn, "Count overflow.\n"
"Considering a random subset of %zu of an unknown and large number of linkages\n",
opts->linkage_limit);
}
if (sent->num_linkages_found == 0)
{
sent->num_linkages_alloced = 0;
sent->num_linkages_post_processed = 0;
sent->num_valid_linkages = 0;
sent->lnkages = NULL;
return;
}
sent->num_linkages_alloced =
MIN(sent->num_linkages_found, (int) opts->linkage_limit);
/* Now actually malloc the array in which we will process linkages. */
/* We may have been called before, e.g. this might be a panic parse,
* and the linkages array may still be there from last time.
* XXX free_linkages() zeros sent->num_linkages_found. */
if (sent->lnkages) free_linkages(sent);
sent->lnkages = linkage_array_new(sent->num_linkages_alloced);
}
/**
* Print the chosen_disjuncts words.
* This is used for debug, e.g. for tracking them in the Wordgraph display.
*/
static void print_chosen_disjuncts_words(const Linkage lkg, bool prt_optword)
{
size_t i;
dyn_str *djwbuf = dyn_str_new();
err_msg(lg_Debug, "Linkage %p (%zu words): ", lkg, lkg->num_words);
for (i = 0; i < lkg->num_words; i++)
{
Disjunct *cdj = lkg->chosen_disjuncts[i];
const char *djw; /* disjunct word - the chosen word */
if (NULL == cdj)
{
djw = (prt_optword && lkg->sent->word[i].optional) ? "{}" : "[]";
}
else if (0 == cdj->is_category)
{
if ('\0' == cdj->word_string[0])
djw = "\\0"; /* null string - something is wrong */
else
djw = cdj->word_string;
}
else
{
if ((NULL == cdj->category))
{
djw = "\\0"; /* something is wrong */
}
else
{
char *cbuf = alloca(32); /* much more space than needed */
snprintf(cbuf, 32, "Category[0]:%u", cdj->category[0].num);
djw = cbuf;
}
}
dyn_strcat(djwbuf, djw);
dyn_strcat(djwbuf, " ");
}
err_msg(lg_Debug, "%s\n", djwbuf->str);
dyn_str_delete(djwbuf);
}
/**
* Return \c true iff \p sent has an optional word.
*/
static bool optional_word_exists(Sentence sent)
{
for (WordIdx w = 0; w < sent->length; w++)
if (sent->word[w].optional) return true;
return false;
}
#define D_PL 7
/**
* This fills the linkage array with morphologically-acceptable
* linkages.
*/
static void process_linkages(Sentence sent, extractor_t* pex,
Parse_Options opts)
{
if (0 == sent->num_linkages_found) return;
if (0 == sent->num_linkages_alloced) return; /* Avoid a later crash. */
/* Pick random linkages if we get more than what was asked for. */
bool pick_randomly = sent->overflowed ||
(sent->num_linkages_found > (int) opts->linkage_limit);
sent->num_valid_linkages = 0;
size_t N_invalid_morphism = 0;
int itry = 0;
size_t in = 0;
int maxtries;
/* In the case of overflow, which will happen for some long
* sentences, but is particularly common for the amy/ady random
* splitters, we want to find as many morpho-acceptable linkages
* as possible, but keep the CPU usage down, as these might be
* very rare. This is due to a bug/feature in the interaction
* between the word-graph and the parser: valid morph linkages
* can be one-in-a-thousand.. or worse. Search for them, but
* don't over-do it.
* Note: This problem has recently been alleviated by an
* alternatives-compatibility check in the fast matcher - see
* alt_connection_possible().
*/
#define MAX_TRIES 250000
if (pick_randomly)
{
/* Try picking many more linkages, but not more than possible. */
maxtries = MIN((int) sent->num_linkages_alloced + MAX_TRIES,
sent->num_linkages_found);
}
else
{
maxtries = sent->num_linkages_alloced;
}
bool need_sane_morphism = !IS_GENERATION(sent->dict) ||
optional_word_exists(sent);
bool need_init = true;
for (itry=0; itry<maxtries; itry++)
{
Linkage lkg = &sent->lnkages[in];
Linkage_info * lifo = &lkg->lifo;
/* Negative values tell extract-links to pick randomly; for
* reproducible-rand, the actual value is the rand seed. */
lifo->index = pick_randomly ? -(itry+1) : itry;
if (need_init)
{
partial_init_linkage(sent, lkg, sent->length);
need_init = false;
}
extract_links(pex, lkg);
compute_link_names(lkg, sent->string_set);
if (verbosity_level(+D_PL))
{
err_msg(lg_Debug, "chosen_disjuncts before:\n\\");
print_chosen_disjuncts_words(lkg, /*prt_opt*/true);
}
if (need_sane_morphism)
{
if (sane_linkage_morphism(sent, lkg, opts))
{
remove_empty_words(lkg);
if (verbosity_level(+D_PL))
{
err_msg(lg_Debug, "chosen_disjuncts after:\n\\");
print_chosen_disjuncts_words(lkg, /*prt_opt*/false);
}
}
else
{
N_invalid_morphism++;
lkg->num_links = 0;
lkg->num_words = sent->length;
// memset(lkg->link_array, 0, lkg->lasz * sizeof(Link));
memset(lkg->chosen_disjuncts, 0, sent->length * sizeof(Disjunct *));
continue;
}
}
if (IS_GENERATION(sent->dict))
compute_generated_words(sent, lkg);
need_init = true;
in++;
if (in >= sent->num_linkages_alloced) break;
}
/* The last one was alloced, but never actually used. Free it. */
if (!need_init) free_linkage(&sent->lnkages[in]);
sent->num_valid_linkages = in;
/* The remainder of the array is garbage; we never filled it in.
* So just pretend that it's shorter than it is */
sent->num_linkages_alloced = sent->num_valid_linkages;
if (verbosity >= D_USER_INFO)
{
lgdebug(0, "Info: sane_morphism(): %zu of %d linkages had "
"invalid morphology construction\n", N_invalid_morphism,
itry + (itry != maxtries));
}
}
/**
* Linkage-equivalent predicate. Return zero if they are equivalent,
* else return +1 or -1. This does provide a stable sort; inequivalent
* linkages are always sorted the same way.
*
* This assumes that the more basic inequivalence compares have already
* been done. This only disambiguates the final little bit of
* nearly-identical linkages.
*/
static int linkage_equiv_p(Linkage lpv, Linkage lnx)
{
// Compare link endpoints
for (uint32_t li=0; li<lpv->num_links; li++)
{
Link * plk = &lpv->link_array[li];
Link * nlk = &lnx->link_array[li];
// Compare word-endpoints first. Most differences are likely
// to be noticeable here. This is an inexpensive check.
int lwd = plk->lw - nlk->lw;
if (lwd) return lwd;
int rwd = plk->rw - nlk->rw;
if (rwd) return rwd;
}
// Compare link names. This is slightly more expensive than the
// check above, so we defer this check.
for (uint32_t li=0; li<lpv->num_links; li++)
{
Link * plk = &lpv->link_array[li];
Link * nlk = &lnx->link_array[li];
// Note (see intersect_strings()):
// link_name is not always in the same string set, so inequality
// test cannot be done here.
if (plk->link_name == nlk->link_name) continue;
int lncmp = strcmp(plk->link_name, nlk->link_name);
if (lncmp) return lncmp;
}
// Compare words. The chosen_disjuncts->word_string is the
// dictionary word. It can happen that two different dictionary
// words can have the same disjunct, and thus result in the same
// linkage. For backwards compat, we will report these as being
// different, as printing will reveal the differences in words.
for (uint32_t wi=0; wi<lpv->num_words; wi++)
{
Disjunct * pdj = lpv->chosen_disjuncts[wi];
Disjunct * ndj = lnx->chosen_disjuncts[wi];
// Parses with non-zero null count will have null words,
// i.e. word without chosen_disjuncts. Avoid a null-pointer
// deref in this case.
if (NULL == pdj)
{
// If one is null, both should be null. (I think this
// will always be true, but I'm not sure.)
if (NULL == ndj) continue;
return 1;
}
// Note (see build_word_expressions()):
// word_string is not always in the same string set, so inequality
// test cannot be done here.
if (pdj->word_string == ndj->word_string) continue;
int wscmp = strcmp(pdj->word_string, ndj->word_string);
if (wscmp) return wscmp;
}
// Compare connector types at the link endpoints. If we are here,
// then the link endpoints landed on the same words, and the link
// names were the same. The connector types might still differ,
// due to intersection. The multi-connector flag might differ.
// However, neither of these are likely. It is plausible to skip
// this check entirely, it's mostly a CPU-time-waster that will
// never find any differences for the almost any situation.
for (uint32_t li=0; li<lpv->num_links; li++)
{
Link * plk = &lpv->link_array[li];
Link * nlk = &lnx->link_array[li];
if (plk->lc != nlk->lc)
{
if (plk->lc->desc != nlk->lc->desc)
return strcmp(connector_string(plk->lc), connector_string(nlk->lc));
int md = plk->lc->multi - nlk->lc->multi;
if (md) return md;
}
if (plk->rc != nlk->rc)
{
if (plk->rc->desc != nlk->rc->desc)
return strcmp(connector_string(plk->rc), connector_string(nlk->rc));
int md = plk->rc->multi - nlk->rc->multi;
if (md) return md;
}
}
#if DOUBLE_CHECK
// We also expect the chosen disjuncts to be identical. But after
// the above checks, it should be impossible that they differ.
for (uint32_t wi=0; wi<lpv->num_words; wi++)
{
if (lpv->chosen_disjuncts[wi] != lnx->chosen_disjuncts[wi])
return strcmp(
linkage_get_disjunct_str(lpv, wi),
linkage_get_disjunct_str(lnx, wi));
}
#endif
// Since the above performed a stable compare, we can safely mark
// the second linkage as a duplicate of the first.
lnx->dupe = true;
return 0;
}
/**
* VDAL == Compare by Violations, Disjunct, Link length.
*/
int VDAL_compare_linkages(Linkage l1, Linkage l2)
{
Linkage_info * p1 = &l1->lifo;
Linkage_info * p2 = &l2->lifo;
if (p1->N_violations != p2->N_violations)
return (p1->N_violations - p2->N_violations);
if (p1->unused_word_cost != p2->unused_word_cost)
return (p1->unused_word_cost - p2->unused_word_cost);
float diff = p1->disjunct_cost - p2->disjunct_cost;
#define COST_EPSILON 1.0e-6
if (COST_EPSILON < diff) return 1;
if (diff < -COST_EPSILON) return -1;
if (p1->link_cost != p2->link_cost)
return (p1->link_cost - p2->link_cost);
if (l1->num_words != l2->num_words)
return l1->num_words - l2->num_words;
// Don't bother sorting bad linkages any further.
if (0 < p1->N_violations) return 0;
return linkage_equiv_p(l1, l2);
}
/**
* Remove duplicate linkages in the link array. Duplicates can appear
* if the number of parses overflowed, or if the number of parses is
* larger than the linkage array. In this case, random linkages will
* be selected, and, by random chance, duplicate linkages can be
* selected. When the alloc array is slightly less than the number of
* linkages found, then as many as half(!) of the linkages can be
* duplicates.
*
* This assumes that the duplicates have already been detected and
* marked by setting `linkage->dupe=true` during linkage sorting.
*/
static void deduplicate_linkages(Sentence sent, int linkage_limit)
{
int linkage_dedup = -1;
const char *test_linkage_dedup = test_enabled("linkage-dedup");
/* Never dedup: linkage-dedup:0; Always dedup: linkage-dedup:1 . */
if (test_linkage_dedup != NULL)
{
if ((test_linkage_dedup[0] != ':') || (test_linkage_dedup[1] == '\0'))
linkage_dedup = 1; /* just linkage-dedup w/o value defaults to 1 */
else
linkage_dedup = atoi(test_linkage_dedup + 1);
}
/* No need for deduplication, if random selection wasn't done. */
if ((linkage_dedup == 0) || ((linkage_dedup < 0) &&
!sent->overflowed && (sent->num_linkages_found <= linkage_limit)))
return;
// Deduplicate the valid linkages only; it's not worth wasting
// CPU time on the rest. Sorting guarantees that the valid
// linkages come first.
uint32_t nl = sent->num_valid_linkages;
if (2 > nl) return;
// Sweep away duplicates
uint32_t tgt = 0;
uint32_t blkstart = 0;
uint32_t blklen = 1; // Initial block, already skipped
uint32_t num_dupes = 0;
for (uint32_t i=1; i<nl; i++)
{
Linkage lnx = &sent->lnkages[i];
if (false == lnx->dupe) { blklen++; continue; }
free_linkage(lnx);
num_dupes ++;
// If there's a block of good linkages to copy, then copy.
if (0 < blklen)
{
// Skip initial block; it is already in place.
if (0 < tgt)
{
Linkage ltgt = &sent->lnkages[tgt];
Linkage lsrc = &sent->lnkages[blkstart];
memmove(ltgt, lsrc, blklen * sizeof(struct Linkage_s));
}
tgt += blklen;
blklen = 0;
}
// The next good linkage comes after this bad one.
blkstart = i+1;
}
// Copy the final block. This will copy the rest of the valid
// linkages, as well as the bad ones. (We need to copy the bad
// ones, because users can still examine them with the UI.)
if (0 < tgt)
{
Linkage ltgt = &sent->lnkages[tgt];
Linkage lsrc = &sent->lnkages[blkstart];
blklen += sent->num_linkages_alloced - sent->num_valid_linkages;
memmove(ltgt, lsrc, blklen * sizeof(struct Linkage_s));
}
assert(num_dupes < sent->num_valid_linkages, "Too many duplicates found!");
// Adjust the totals.
sent->num_linkages_alloced -= num_dupes;
sent->num_valid_linkages -= num_dupes;
sent->num_linkages_post_processed -= num_dupes;
}
static void sort_linkages(Sentence sent, Parse_Options opts)
{
if (0 == sent->num_linkages_found) return;
/* It they're randomized, don't bother sorting */
if (0 != sent->rand_state && sent->dict->shuffle_linkages) return;
/* Initialize all linkages as unique */
for (uint32_t i=0; i<sent->num_linkages_alloced; i++)
sent->lnkages[i].dupe = false;
/* Sorting will also mark some of them as being duplicates */
qsort((void *)sent->lnkages, sent->num_linkages_alloced,
sizeof(struct Linkage_s),
(int (*)(const void *, const void *))opts->cost_model.compare_fn);
/* Remove the duplicates. */
deduplicate_linkages(sent, opts->linkage_limit);
print_time(opts, "Sorted all linkages");
}
static void notify_no_complete_linkages(unsigned int null_count,
unsigned int max_null_count)
{
if ((0 == null_count) && (0 < max_null_count) && verbosity > 0)
prt_error("No complete linkages found.\n");
}
/**
* classic_parse() -- parse the given sentence.
* Perform parsing, using the original link-grammar parsing algorithm
* given in the original link-grammar papers.
*
* Do the parse with the minimum number of null-links within the range
* specified by opts->min_null_count and opts->max_null_count.
*
* To that end, call do_parse() with an increasing null_count, from
* opts->min_null_count up to (including) opts->max_null_count, until a
* parse is found.
*
* To increase the parsing speed, before invoking do_parse(), invoke
* pp_and_power_prune() to remove connectors which have no possibility to
* connect. Since power_prune() includes a significant optimization if it
* assumes that the linkage has no more than a specific number of null
* links (aka null_count), call it with the current number of null_count
* for which do_count() is invoked.
*
* In order to be able to so repeat the pruning step, we need to keep
* the original disjunct/connectors in order to prune them again.
* This is done only when needed, i.e. when we are invoked with
* min_null_count != max_null_count (a typical case is that they are
* both 0).
*
* So in case this optimization has been done and a parse (e.g.
* a parse when null_count==0) is not found, we are left with sentence
* disjuncts which are not appropriate to continue do_parse() tries with
* a greater null_count. To solve that, we need to restore the original
* disjuncts of the sentence and call pp_and_power_prune() once again.
*/
void classic_parse(Sentence sent, Parse_Options opts)
{
fast_matcher_t * mchxt = NULL;
count_context_t * ctxt = NULL;
Tracon_sharing *ts_parsing = NULL;
void *saved_memblock = NULL;
int current_prune_level = -1; /* -1: No pruning has been done yet. */
int needed_prune_level = opts->min_null_count;
bool more_pruning_possible = false;
unsigned int max_null_count = opts->max_null_count;
max_null_count = (unsigned int)MIN(max_null_count, sent->length);
bool one_step_parse = (unsigned int)opts->min_null_count != max_null_count;
int max_prune_level = (int)max_null_count;
bool optimize_pruning = true; /* Perform pruning null count optimization. */
unsigned int *ncu[2];
ncu[0] = alloca(sent->length * sizeof(*ncu[0]));
ncu[1] = alloca(sent->length * sizeof(*ncu[1]));
/* Null-count optimization not implemented for islands_ok==true. */
if (opts->islands_ok)
optimize_pruning = false;
/* Pruning per null-count and one-step-parse are costly for sentences
* whose parsing takes tens of milliseconds or so. Disable them for
* short-enough sentences. */
if (sent->length < sent->min_len_multi_pruning)
optimize_pruning = false;
if (!optimize_pruning)
{
/* Turn-off null-count optimization. */
if (opts->min_null_count == 0)
max_prune_level = 0;
else
{
needed_prune_level = MAX_SENTENCE;
one_step_parse = false;
}
}
/* Build lists of disjuncts */
prepare_to_parse(sent, opts);
if (resources_exhausted(opts->resources)) return; /* Nothing to free yet. */
Tracon_sharing *ts_pruning = pack_sentence_for_pruning(sent);
free_sentence_disjuncts(sent, /*category_too*/false);
if (one_step_parse)
{
/* Save the disjuncts for possible parse w/ an increased null count. */
saved_memblock = save_disjuncts(sent, ts_pruning);
}
print_time(opts, "Encoded for pruning%s%s",
(NULL == ts_pruning->tracon_list) ? " (skipped)" : "",
(one_step_parse) ? " (one-step)" : "");
for (unsigned int nl = opts->min_null_count; nl <= max_null_count; nl++)
{
sent->null_count = nl;
/* We may be here again for parsing with a higher null_count since
* num_valid_linkages of the previous parse was 0 because all the
* linkages had P.P. violations. Ensure that in case of a timeout we
* will not end up with the previous num_linkages_found. */
sent->num_linkages_found = 0;
sent->overflowed = false;
sent->num_valid_linkages = 0;
sent->num_linkages_post_processed = 0;
if (needed_prune_level > current_prune_level)
{
current_prune_level = needed_prune_level;
if (needed_prune_level < max_prune_level)
needed_prune_level++;
else
needed_prune_level = MAX_SENTENCE;
if (more_pruning_possible)
restore_disjuncts(sent, saved_memblock, ts_pruning);
more_pruning_possible =
one_step_parse && (current_prune_level != MAX_SENTENCE);
unsigned int expected_null_count =
pp_and_power_prune(sent, ts_pruning, current_prune_level, opts,
ncu);
if (expected_null_count > nl)
{
if (opts->verbosity >= D_USER_TIMES)
{
prt_error("#### Skip parsing (w/%u ", nl);
if (expected_null_count-1 > nl)
prt_error("to %u nulls)\n", expected_null_count-1);
else
prt_error("null%s)\n", (nl != 1) ? "s" : "");
}
notify_no_complete_linkages(nl, max_null_count);
nl = expected_null_count-1;
/* To get a result, parse w/null count which is at most one less
* than the number of tokens (w/all nulls there is no linkage). */
if (nl == sent->length-1) nl--;
continue;
}
}
if (NULL != ts_pruning)
{
free_tracon_sharing(ts_parsing);
ts_parsing = pack_sentence_for_parsing(sent);
print_time(opts, "Encoded for parsing");
if (!more_pruning_possible)
{
/* At this point no further pruning will be done. Free the
* pruning tracon stuff here instead of at the end. */
free_tracon_memblock(ts_pruning);
ts_pruning = NULL;
if (NULL != saved_memblock)
free_saved_memblock(saved_memblock);
}
gword_record_in_connector(sent);
free_fast_matcher(sent, mchxt);
mchxt = alloc_fast_matcher(sent, ncu);
print_time(opts, "Initialized fast matcher");
if (resources_exhausted(opts->resources)) goto parse_end_cleanup;
}
free_linkages(sent);
free_count_context(ctxt, sent);
ctxt = alloc_count_context(sent, ts_parsing);
sent->num_linkages_found = do_parse(sent, mchxt, ctxt, opts);
print_time(opts, "Counted parses (%d w/%u null%s)",
sent->num_linkages_found, sent->null_count,
(sent->null_count != 1) ? "s" : "");
/* In case of a timeout, the linkage is partial and may be
* inconsistent. It is also usually different on each run.
* So in that case, pretend that the linkage count is 0. */
if (resources_exhausted(opts->resources))
{
sent->num_linkages_found = 0;
goto parse_end_cleanup;
}
if (sent->num_linkages_found > 0)
{
extractor_t * pex = extractor_new(sent);
setup_linkages(sent, pex, mchxt, ctxt, opts);
process_linkages(sent, pex, opts);
if (IS_GENERATION(sent->dict))
find_unused_disjuncts(sent, pex);
#ifdef PC_DISPLAY
display_parse_choice(pex);
#endif
free_extractor(pex);
post_process_lkgs(sent, opts);
if (resources_exhausted(opts->resources))
{
sent->num_linkages_found = 0;
sent->num_valid_linkages = 0;
sent->num_linkages_post_processed = 0;
goto parse_end_cleanup;
}
if (sent->num_valid_linkages > 0) break;
if (verbosity >= D_USER_INFO)
{
/* FIXME:
* 1. Issue this message if verbosity != 0.
* 2. Don't continue parsing with higher null counts. */
if ((sent->num_linkages_post_processed > 0) &&
(sent->num_linkages_post_processed == sent->num_linkages_alloced) &&
((int)opts->linkage_limit < sent->num_linkages_found) &&
!IS_GENERATION(sent->dict))
prt_error("Info: All examined linkages (%zu) had P.P. violations.\n"
"Consider increasing the linkage limit.\n"
"At the command line, use !limit\n",
sent->num_linkages_post_processed);
}
}
notify_no_complete_linkages(nl, max_null_count);
}
if ((sent->num_linkages_found == 0) && IS_GENERATION(sent->dict))
find_unused_disjuncts(sent, NULL);
sort_linkages(sent, opts);
parse_end_cleanup:
if (NULL != ts_pruning)
{
free_categories(sent);
free_tracon_memblock(ts_pruning);
free_saved_memblock(saved_memblock);
}
free_tracon_sharing(ts_parsing);
free_count_context(ctxt, sent);
free_fast_matcher(sent, mchxt);
}