/*************************************************************************/
/* Copyright (c) 2004 */
/* Daniel Sleator, David Temperley, and John Lafferty */
/* Copyright 2013, 2014 Linas Vepstas */
/* All rights reserved */
/* */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software. */
/* This license allows free redistribution and use in source and binary */
/* forms, with or without modification, subject to certain conditions. */
/* */
/*************************************************************************/
#include <ctype.h>
#include <inttypes.h> // format macros
#include <math.h> // fabsf signbit
#include "api-structures.h" // Parse_Options_s (seems hacky to me)
#include "dict-common.h"
#include "dict-defines.h"
#include "dict-file/word-file.h"
#include "dict-file/read-dict.h"
#include "dict-utils.h" // copy_Exp
#include "disjunct-utils.h"
#include "prepare/build-disjuncts.h" // build_disjuncts_for_exp
#include "print/print.h"
#include "print/print-util.h"
#include "regex-morph.h"
#include "tokenize/tokenize.h" // word_add
#include "tokenize/word-structures.h" // Word_struct
#include "utilities.h" // GNUC_UNUSED
/* ======================================================================== */
bool cost_eq(float cost1, float cost2)
{
return (fabsf(cost1 - cost2) < cost_epsilon);
}
/* Create a float constant with mantissa a and exponent s. */
#define COMBINE(a, b, c) a ## b ## c
#define EXP10F(a, s) COMBINE(a ## e, s, F)
/**
* Convert \p cost to a string with COST_MAX_DEC_PLACES decimal places.
* Always use dot as a radix character.
* @return A static thread-local pointer to the result string.
*/
const char *cost_stringify(float cost)
{
static TLS char buf[COST_MAX_DEC_PLACES+8];
static const float scale = EXP10F(1, COST_MAX_DEC_PLACES);
bool sign = signbit(cost);
float ac = fabsf(cost);
int c = (int) floorf(ac);
unsigned long r = (unsigned long) roundf((ac-c)*scale);
int l = snprintf(buf, sizeof(buf), "%s%d.%0*lu", sign ? "-" : "",
c, COST_MAX_DEC_PLACES, r);
if ((l < 0) || (l >= (int)sizeof(buf))) return "ERR_COST";
return buf;
}
/* Check for an existing newline before issuing "\n" in order to prevent
* empty lines when printing connector macros. This allows to use the
* same print_expression_tag_*() function for printing expressions
* with macros and also disjunct connectors with macros. */
static void dyn_ensure_empty_line(dyn_str *e)
{
if (dyn_strlen(e) > 0)
{
dyn_trimback(e);
if ((dyn_str_value(e)[dyn_strlen(e)-1]) != '\n')
dyn_strcat(e, "\n");
}
}
#define MACRO_INDENTATION 4
static void print_expression_tag_start(Dictionary dict, dyn_str *e, const Exp *n,
int *indent)
{
if (n->type == CONNECTOR_type) return;
if (NULL == dict) return;
switch (n->tag_type)
{
case Exptag_none:
break;
case Exptag_dialect:
dyn_strcat(e, "[");
break;
case Exptag_macro:
if (*indent < 0) break;
dyn_ensure_empty_line(e);
for(int i = 0; i < *indent; i++) dyn_strcat(e, " ");
dyn_strcat(e, dict->macro_tag->name[n->tag_id]);
dyn_strcat(e, ": ");
*(indent) += MACRO_INDENTATION;
break;
default:
for(int i = 0; i < *indent; i++) dyn_strcat(e, " ");
append_string(e, "Unknown tag type %d: ", (int)n->tag_type);
*(indent) += MACRO_INDENTATION;
}
}
static void print_expression_tag_end(Dictionary dict, dyn_str *e, const Exp *n,
int *indent)
{
if (n->type == CONNECTOR_type) return;
if (NULL == dict) return;
switch (n->tag_type)
{
case Exptag_none:
break;
case Exptag_dialect:
dyn_strcat(e, "]");
dyn_strcat(e, dict->dialect_tag.name[n->tag_id]);
break;
case Exptag_macro:
if (*indent < 0) break;
dyn_ensure_empty_line(e);
for(int i = 0; i < *indent - MACRO_INDENTATION/2; i++)
dyn_strcat(e, " ");
(*indent) -= MACRO_INDENTATION;
break;
default:
/* Handled in print_expression_tag_start(). */
;
}
}
static void get_expression_cost(const Exp *e, unsigned int *icost, float *dcost)
{
if (e->cost < -cost_epsilon)
{
*icost = 1;
*dcost = e->cost;
}
else if (cost_eq(e->cost, 0.0))
{
/* avoid [X+]-0.00 */
*icost = 0;
*dcost = 0;
}
else
{
*icost = (int) (e->cost);
*dcost = e->cost - *icost;
if (*dcost > cost_epsilon)
{
*dcost = e->cost;
*icost = 1;
}
else
{
if (*icost > 4)
{
/* don't print too many [] levels */
*dcost = (float)*icost;
*icost = 1;
}
else
{
*dcost = 0;
}
}
}
}
static bool is_expression_optional(const Exp *e)
{
Exp *o = e->operand_first;
return (e->type == OR_type) && (o != NULL) && (o->type == AND_type) &&
(NULL == o->operand_first) && (o->cost == 0) &&
(o->tag_type == Exptag_none);
}
static void print_expression_parens(Dictionary dict, dyn_str *e, const Exp *n,
bool need_parens, int *indent)
{
unsigned int icost;
float dcost;
get_expression_cost(n, &icost, &dcost);
for (unsigned int i = 0; i < icost; i++) dyn_strcat(e, "[");
print_expression_tag_start(dict, e, n, indent);
const char *opr = NULL;
Exp *opd = n->operand_first;
if (n->type == CONNECTOR_type)
{
if (n->multi) dyn_strcat(e, "@");
dyn_strcat(e, n->condesc ? n->condesc->more->string : "error-null-connector");
dyn_strcat(e, (const char []){ n->dir, '\0' });
}
else if (is_expression_optional(n))
{
dyn_strcat(e, "{");
if (NULL == opd->operand_next)
dyn_strcat(e, "error-no-next"); /* unary OR */
else
print_expression_parens(dict, e, opd->operand_next, false, indent);
dyn_strcat(e, "}");
}
else
{
if (n->type == AND_type)
opr = " & ";
else if (n->type == OR_type)
opr = " or ";
else
append_string(e, "error-exp-type-%d", (int)n->type);
if (opr != NULL)
{
/* (opd == NULL) means this is a null expression. */
if (((icost == 0) && need_parens) || (opd == NULL)) dyn_strcat(e, "(");
if ((opd == NULL) && (n->type == OR_type))
dyn_strcat(e, "error-zeroary-or");
for (Exp *l = opd; l != NULL; l = l->operand_next)
{
print_expression_parens(dict, e, l, true, indent);
if (l->operand_next != NULL)
dyn_strcat(e, opr);
else if ((n->type == OR_type) && (l == n->operand_first))
dyn_strcat(e, " or error-no-next"); /* unary OR */
}
if (((icost == 0) && need_parens) || (opd == NULL)) dyn_strcat(e, ")");
}
}
for (unsigned int i = 0; i < icost; i++) dyn_strcat(e, "]");
if (dcost != 0) dyn_strcat(e, cost_stringify(dcost));
print_expression_tag_end(dict, e, n, indent);
}
/**
* Find if the given connector is in the given expression.
* @param e Expression.
* @param find_pos Connector position to search in \p e.
* @param pos Temporary connector position while the search advances.
* Return \c true iff the connector is found.
*/
static bool exp_contains_connector(const Exp *e, int *pos, int find_pos)
{
if (NULL == e) return false;
if (CONNECTOR_type == e->type)
{
#if 0
printf("exp_contains_connector: pos=%d C=%s%s%c %s\n",
*pos,e->multi?"@":"",e->condesc->more->string,e->dir,
(find_pos == *pos) ? "FOUND" : "");
#endif
return (find_pos == (*pos)++);
}
for(Exp *opd = e->operand_first; opd != NULL; opd = opd->operand_next)
{
if (exp_contains_connector(opd, pos, find_pos))
return true;
}
return false;
}
typedef struct
{
Dictionary dict;
dyn_str *e;
int indent;
int pos; /* current connector position in expression */
int *find_pos; /* disjunct expression connectors positions */
bool is_after_connector; /* an indicator for printing '&' */
} cmacro_context;
/**
* Print nested macros for each desired connector.
* The desired connector positions are in find_pos[] (ascending sorted,
* -1 terminated). Its first element holds the next position of the next
* connector to print, and is chopped after the connector is printed.
*/
static void print_connector_macros(cmacro_context *cmc, const Exp *n)
{
if ((cmc->find_pos)[0] == -1)
return; /* fast termination when nothing more to do */
bool macro_started = false;
int current_pos = cmc->pos;
if ((Exptag_macro == n->tag_type) &&
exp_contains_connector(n, ¤t_pos, (cmc->find_pos)[0]))
{
if (cmc->is_after_connector)
{
dyn_strcat(cmc->e, " & ");
cmc->is_after_connector = false;
}
print_expression_tag_start(cmc->dict, cmc->e, n, &cmc->indent);
macro_started = true;
}
Exp *opd = n->operand_first;
if (n->type == CONNECTOR_type)
{
if ((cmc->find_pos)[0] == cmc->pos)
{
if (cmc->is_after_connector) dyn_strcat(cmc->e, " & ");
cmc->is_after_connector = true;
if (n->multi) dyn_strcat(cmc->e, "@");
dyn_strcat(cmc->e,
n->condesc ? n->condesc->more->string : "error-null-connector");
dyn_strcat(cmc->e, (const char []){ n->dir, '\0' });
cmc->find_pos++; /* each expression position is used only once */
}
cmc->pos++;
}
else
{
for (Exp *l = opd; l != NULL; l = l->operand_next)
{
print_connector_macros(cmc, l);
}
}
/* The -1 check is to suppress unneeded newlines at the end. */
if (macro_started && ((cmc->find_pos)[0] != -1))
print_expression_tag_end(cmc->dict, cmc->e, n, &cmc->indent);
}
/* Note: The returned value is to be freed by the caller. */
static char *lg_exp_stringify_with_tags(Dictionary dict, const Exp *n,
bool show_macros)
{
if (n == NULL) return strdup("(null)");
int indent = show_macros ? 0 : -1;
dyn_str *e = dyn_str_new();
print_expression_parens(dict, e, n, false, &indent);
return dyn_str_take(e);
}
/**
* Stringify the given expression, ignoring tags.
* Note: The returned value is to be freed by the caller.
*/
char *lg_exp_stringify(const Exp *n)
{
return lg_exp_stringify_with_tags(NULL, n, false);
}
/* For usage convenience when assigning to a temporary variable + free() is
* cumbersome. Care should be taken due to the static memory of the result. */
const char *exp_stringify(const Exp *n)
{
static TLS char *s = NULL;
free(s);
s = NULL;
if (n == NULL) return ("(null)");
s = lg_exp_stringify_with_tags(NULL, n, false);
return s;
}
#ifdef DEBUG
/* There is a much better lg_exp_stringify() elsewhere
* This one is for low-level debug. */
GNUC_UNUSED void prt_exp(Exp *e, int i)
{
if (e == NULL) return;
for(int j =0; j<i; j++) printf(" ");
printf ("type=%d dir=%c multi=%d cost=%s\n",
e->type, e->dir, e->multi, cost_stringify(e->cost));
if (e->type != CONNECTOR_type)
{
for (e = e->operand_next; e != NULL; e = e->operand_next) prt_exp(e, i+2);
}
else
{
for(int j =0; j<i; j++) printf(" ");
printf("con=%s\n", e->condesc->more->string);
}
}
#endif
static const char *stringify_Exp_type(Exp_type type)
{
static TLS char unknown_type[32] = "";
const char *type_str;
if (type > 0 && type <= 3)
{
type_str = ((const char *[]) {"OR", "AND", "CONNECTOR"}) [type-1];
}
else
{
snprintf(unknown_type, sizeof(unknown_type), "unknown_type-%d",
(int)(type));
type_str = unknown_type;
}
return type_str;
}
static const char *stringify_Exp_tag(Exp *e, Dictionary dict)
{
static TLS char tag_info[64];
if (e->type == CONNECTOR_type) return "";
switch (e->tag_type)
{
case Exptag_none:
return "";
case Exptag_dialect:
if (dict == NULL)
{
snprintf(tag_info, sizeof(tag_info), " dialect_tag=%u",
(unsigned int)e->tag_id);
}
else
{
snprintf(tag_info, sizeof(tag_info), " dialect_tag=%s",
dict->dialect_tag.name[e->tag_id]);
}
break;
case Exptag_macro:
if (dict == NULL)
{
snprintf(tag_info, sizeof(tag_info), " macro_tag");
}
else
{
snprintf(tag_info, sizeof(tag_info), " macro_tag=%s",
dict->macro_tag->name[e->tag_id]);
}
break;
default:
snprintf(tag_info, sizeof(tag_info), " unknown_tag_type-%d",
(int)(e->tag_type));
;
}
return tag_info;
}
static bool is_ASAN_uninitialized(uintptr_t a)
{
static const uintptr_t asan_uninitialized = (uintptr_t)0xbebebebebebebebeULL;
return (a == asan_uninitialized);
}
static void prt_exp_all(dyn_str *s, Exp *e, int i, Dictionary dict)
{
if (is_ASAN_uninitialized((uintptr_t)e))
{
dyn_strcat(s, "e=UNINITIALIZED\n");
return;
}
if (e == NULL) return;
for(int j =0; j<i; j++) dyn_strcat(s, " ");
append_string(s, "e=%p: %s", e, stringify_Exp_type(e->type));
if (is_ASAN_uninitialized((uintptr_t)e->operand_first))
dyn_strcat(s, " (UNINITIALIZED operand_first)");
if (is_ASAN_uninitialized((uintptr_t)e->operand_next))
dyn_strcat(s, " (UNINITIALIZED operand_next)");
if (e->type != CONNECTOR_type)
{
int operand_count = 0;
for (Exp *opd = e->operand_first; NULL != opd; opd = opd->operand_next)
{
operand_count++;
if (is_ASAN_uninitialized((uintptr_t)opd->operand_next))
{
append_string(s, " (operand %d: UNINITIALIZED operand_next)\n",
operand_count);
return;
}
}
append_string(s, " (%d operand%s) cost=%s%s\n", operand_count,
operand_count == 1 ? "" : "s", cost_stringify(e->cost),
stringify_Exp_tag(e, dict));
for (Exp *opd = e->operand_first; NULL != opd; opd = opd->operand_next)
{
prt_exp_all(s, opd, i+2, dict);
}
}
else
{
append_string(s, " %s%s%c cost=%s%s\n",
e->multi ? "@" : "",
e->condesc ? e->condesc->more->string : "(condesc=(null))",
e->dir, cost_stringify(e->cost),
stringify_Exp_tag(e, dict));
}
}
#ifdef DEBUG
GNUC_UNUSED void prt_exp_mem(Exp *e)
{
dyn_str *s = dyn_str_new();
prt_exp_all(s, e, 0, NULL);
char *e_str = dyn_str_take(s);
printf("%s", e_str);
free(e_str);
}
#endif
/* ================ Print disjuncts and connectors ============== */
static bool is_flag(uint32_t flags, char flag)
{
return (flags>>(flag-'a')) & 1;
}
static uint32_t make_flags(const char *flags)
{
if (flags == NULL) return 0;
uint32_t r = 0;
for (const char *f = flags; *f != '\0'; f++)
r |= (1<<(*f-'a'));
return r;
}
/* Print one connector with all the details.
* mCnameD<tracon_id>{refcount}(nearest_word, length_limit)x
* Optional m: "@" for multi (else nothing).
* Cname: Connector name.
* Optional D: "-" / "+" (if dir != -1).
* Optional <tracon_id>: (flag 't').
* Optional [nearest_word, length_limit or farthest_word]: (flag 'l').
* x: Shallow/deep indication as "s" / "d"
*/
static void dyn_print_one_connector(dyn_str *s, const Connector *e, int dir,
uint32_t flags)
{
if (e->multi)
dyn_strcat(s, "@");
dyn_strcat(s, (e->desc != NULL) ? connector_string(e) : "NULLDESC");
if (-1 != dir) dyn_strcat(s, (dir == 0) ? "-" : "+");
if (is_flag(flags, 't') && e->tracon_id)
append_string(s, "<%d>", e->tracon_id);
if (is_flag(flags, 'r') && e->refcount)
append_string(s, "{%d}",e->refcount);
if (is_flag(flags, 'l'))
append_string(s, "(%d,%d)", e->nearest_word, e->farthest_word);
#if 0
append_string(s, "<<%d>>", e->exp_pos);
#endif
if (is_flag(flags, 's'))
dyn_strcat(s, e->shallow ? "s" : "d");
}
void print_one_connector(const Connector *e, const char *flags)
{
dyn_str *s = dyn_str_new();
int dir = -1;
if (flags == NULL) flags = "lt";
if (*flags == '-') { flags++; dir = 0; }
if (*flags == '+') { flags++; dir = 1; }
uint32_t int_flags = make_flags(flags);
dyn_print_one_connector(s, e, dir, int_flags);
char *t = dyn_str_take(s);
puts(t);
free(t);
}
static void dyn_print_connector_list(dyn_str *s, const Connector *e, int dir,
uint32_t flags)
{
if (e == NULL) return;
dyn_print_connector_list(s, e->next, dir, flags);
if (e->next != NULL) dyn_strcat(s, " ");
dyn_print_one_connector(s, e, dir, flags);
}
/* Special flags here: Initial "-" or "+" for direction sign. */
char *print_connector_list_str(const Connector *e, const char *flags)
{
dyn_str *s = dyn_str_new();
int dir = -1;
if (flags == NULL) flags = "lt";
if (*flags == '-') { flags++; dir = 0; }
if (*flags == '+') { flags++; dir = 1; }
dyn_print_connector_list(s, e, dir, make_flags(flags));
return dyn_str_take(s);
}
char *print_one_connector_str(const Connector *e, const char *flags)
{
dyn_str *s = dyn_str_new();
int dir = -1;
if (flags == NULL) flags = "lt";
if (*flags == '-') { flags++; dir = 0; }
if (*flags == '+') { flags++; dir = 1; }
uint32_t int_flags = make_flags(flags);
dyn_print_one_connector(s, e, dir, int_flags);
return dyn_str_take(s);
}
/* Ascending sort of connector positions. */
static int ascending_int(const void *a, const void *b)
{
const int a1 = *(const int *)a;
const int b1 = *(const int *)b;
if (a1 < b1) return -1;
if (a1 == b1) return 0;
return 1;
}
typedef struct
{
const Regex_node *regex;
Exp *exp;
Dictionary dict;
unsigned int num_selected;
unsigned int num_tunnels;
} select_data;
static void dyn_print_disjunct_list(dyn_str *s, const Disjunct *dj,
uint32_t flags,
bool (* select)(const char *dj_str, select_data *criterion),
select_data *criterion)
{
int djn = 0;
char word[MAX_WORD + 32];
int max_ccnt = 0;
int *exp_pos = NULL;
bool print_disjunct_address = test_enabled("disjunct-address");
for (;dj != NULL; dj=dj->next)
{
if (dj->is_category == 0)
{
lg_strlcpy(word, dj->word_string, sizeof(word));
}
else
{
int n = snprintf(word, sizeof(word), "%x", dj->category[0].num);
if (dj->num_categories > 1)
snprintf(word + n, sizeof(word) - n, " (%u)", dj->num_categories);
}
patch_subscript_mark(word);
dyn_str *l = dyn_str_new();
append_string(l, "%16s", word);
if (print_disjunct_address) append_string(s, "(%p)", dj);
dyn_strcat(l, ": ");
const char *cost_str = cost_stringify(dj->cost);
append_string(l, "[%d]%s%s= ", djn++, &" "[(*cost_str == '-')], cost_str);
dyn_print_connector_list(l, dj->left, /*dir*/0, flags);
dyn_strcat(l, " <> ");
dyn_print_connector_list(l, dj->right, /*dir*/1, flags);
char *ls = dyn_str_take(l);
if ((NULL == select) || select(ls, criterion))
{
dyn_strcat(s, ls);
dyn_strcat(s, "\n");
if ((criterion != NULL) && (criterion->exp != NULL))
{
int ccnt = 1; /* 1 for exp_pos -1 terminator. */
for (Connector *c = dj->left; c != NULL; c = c->next)
ccnt++;
for (Connector *c = dj->right; c != NULL; c = c->next)
ccnt++;
if (ccnt > max_ccnt)
{
max_ccnt = (ccnt == 0) ? 32 : ccnt;
exp_pos = alloca(max_ccnt * sizeof(int));
}
int *i = exp_pos;
for (Connector *c = dj->left; c != NULL; c = c->next)
*i++ = c->exp_pos;
for (Connector *c = dj->right; c != NULL; c = c->next)
*i++ = c->exp_pos;
*i = -1;
qsort(exp_pos, ccnt-1, sizeof(int), ascending_int);
cmacro_context cmc = {
.dict = criterion->dict,
.e = s,
.find_pos = exp_pos,
};
print_connector_macros(&cmc, criterion->exp);
dyn_strcat(s, "\n\n");
}
}
free(ls);
}
}
void print_disjunct_list(const Disjunct *d, const char *flags)
{
dyn_str *s = dyn_str_new();
if (flags == NULL) flags = "lt";
uint32_t int_flags = make_flags(flags);
dyn_print_disjunct_list(s, d, int_flags, NULL, NULL);
char *t = dyn_str_take(s);
puts(t);
free(t);
}
void print_all_disjuncts(Sentence sent)
{
dyn_str *s = dyn_str_new();
uint32_t int_flags = make_flags("lt");
for (WordIdx w = 0; w < sent->length; w++)
{
append_string(s, "Word %zu:\n", w);
dyn_print_disjunct_list(s, sent->word[w].d, int_flags, NULL, NULL);
}
char *t = dyn_str_take(s);
puts(t);
free(t);
}
/* ================ Display word expressions / disjuncts ================= */
#define DJ_COL_WIDTH sizeof(" ")
static bool select_disjunct(const char *dj_str, select_data *criterion)
{
/* Count number of disjuncts with tunnel connectors. */
for (const char *p = dj_str; *p != '\0'; p++)
{
if ((p[0] == ' ') && (p[1] == 'x'))
{
criterion->num_tunnels++;
break;
}
}
/* Select desired disjuncts. If several connectors need to match in any
* order, the Regex_node list contains more than one component, and all
* of them must match. A horrible hack is done below to achieve that. */
for (Regex_node *rn = (Regex_node *)criterion->regex; rn != NULL;
rn = rn->next)
{
Regex_node *savenext = rn->next;
rn->next = NULL;
if (match_regex(rn , dj_str) == NULL)
{
rn->next = savenext;
return false;
}
rn->next = savenext;
}
criterion->num_selected++;
return true;
}
/**
* Display the disjuncts of expressions in \p dn.
*/
static char *display_disjuncts(Dictionary dict, const Dict_node *dn,
const void **arg)
{
const Regex_node *rn = arg[0];
const char *flags = arg[1];
const Parse_Options opts = (Parse_Options)arg[2];
float max_cost = opts->disjunct_cost;
uint32_t int_flags = make_flags(flags);;
/* build_disjuncts_for_exp() needs memory pools for efficiency. */
Sentence dummy_sent = sentence_create("", dict); /* For memory pools. */
dummy_sent->Disjunct_pool = pool_new(__func__, "Disjunct",
/*num_elements*/8192, sizeof(Disjunct),
/*zero_out*/false, /*align*/false, false);
dummy_sent->Connector_pool = pool_new(__func__, "Connector",
/*num_elements*/65536, sizeof(Connector),
/*zero_out*/true, /*align*/false, false);
select_data criterion = { .regex = rn };
void *select = (rn == NULL) ? NULL : select_disjunct;
dyn_str *s = dyn_str_new();
dyn_strcat(s, "disjuncts:\n");
for (; dn != NULL; dn = dn->right)
{
/* Use copy_Exp() to assign dialect cost. */
Exp *e = copy_Exp(dn->exp, dummy_sent->Exp_pool, opts);
Disjunct *d = build_disjuncts_for_exp(dummy_sent, e, dn->string, NULL,
max_cost, NULL);
unsigned int dnum0 = count_disjuncts(d);
unsigned int dnum1 = dnum0 - eliminate_duplicate_disjuncts(d, false);
if ((flags != NULL) && (strchr(flags, 'm') != NULL))
{
criterion.exp = e;
criterion.dict = dict;
}
criterion.num_selected = 0;
dyn_str *dyn_pdl = dyn_str_new();
dyn_print_disjunct_list(dyn_pdl, d, int_flags, select, &criterion);
char *dliststr = dyn_str_take(dyn_pdl);
pool_reuse(dummy_sent->Exp_pool);
pool_reuse(dummy_sent->Disjunct_pool);
pool_reuse(dummy_sent->Connector_pool);
append_string(s, " %-*s %8u/%u disjuncts",
display_width(DJ_COL_WIDTH, dn->string), dn->string,
dnum1, dnum0);
if (criterion.num_tunnels != 0)
append_string(s, " (%u tunnels)", criterion.num_tunnels);
dyn_strcat(s, "\n\n");
dyn_strcat(s, dliststr);
dyn_strcat(s, "\n");
free(dliststr);
if (rn != NULL)
{
if (criterion.num_selected == dnum1)
dyn_strcat(s, "(all the disjuncts matched)\n\n");
else
append_string(s, "(%u disjunct%s matched)\n\n",
criterion.num_selected,
criterion.num_selected == 1 ? "" : "s");
}
}
sentence_delete(dummy_sent);
return dyn_str_take(s);
}
static void notify_ignoring_flag(const char *flag)
{
if (flag != NULL) prt_error("Warning: Ignoring flag \"%c\".\n", *flag);
}
/**
* Copy bytes from \p src to \p dst, while quoting with \c '\\'
* the bytes in \p q.
* Note: This is not a general-purpose function, since:
* - It assumes \p dst has enough space.
* - \p src is not checked for NUL.
* - No NUL termination is done.
*
* @param src Source buffer.
* @param dst[out] Destination buffer.
* @return Number of bytes added to \p dst.
*/
static size_t copy_quoted(const char *q, char *dst, const char *src, size_t len)
{
const char *orig_dst = dst;
for (; len-- > 0; src++, dst++)
{
if (strchr(q, *src) != NULL) *dst++ = '\\';
*dst = *src;
}
return (size_t)(dst - orig_dst);
}
static Regex_node *new_disjunct_regex_node(Regex_node *current, char *regpat)
{
Regex_node *rn = regex_new("Disjunct regex", regpat);
rn->next = current;
return rn;
}
static Regex_node *make_disjunct_pattern(const char *pattern, const char *flags)
{
if (NULL == flags) flags = "";
const char *is_full = strchr(flags, 'f');
const char *is_regex = strchr(flags, 'r');
const char *is_anyorder = strchr(flags, 'a');
char *regpat; /* constructed regex pattern */
size_t pat_len = strlen(pattern);
Regex_node *rn = NULL;
if ('\0' == pattern[strspn(pattern, "0123456789")])
{
notify_ignoring_flag(is_regex);
notify_ignoring_flag(is_full);
notify_ignoring_flag(is_anyorder);
const size_t added_chars_len = sizeof("\\[]");
regpat = alloca(pat_len + added_chars_len);
strcpy(regpat, "\\[");
strcat(regpat, pattern);
strcat(regpat, "]");
}
else if (is_full != NULL)
{
notify_ignoring_flag(is_regex);
notify_ignoring_flag(is_anyorder);
/* Assume this is the complete specification of the disjunct, e.g.
* as copied from the !disjuncts output. Insert "<>" after the LHS
* connectors and build a search regex. Extra spaces are not
* supported (and are not checked). */
for (size_t i = 0; i < pat_len; i++)
{
const char *c = &pattern[i];
if (!isalnum((unsigned char)*c) && (strchr("*+- ", *c) == NULL))
{
prt_error("Warning: Invalid character \"%.*s\" in full "
"disjunct specification.\n",
(utf8_charlen(c) < 0) ? 0 : utf8_charlen(c), c);
}
}
size_t rhs_pos = strcspn(pattern, "+");
if ('\0' != pattern[rhs_pos])
{
for (rhs_pos--; rhs_pos != 0; rhs_pos--)
if (' ' == pattern[rhs_pos]) break;
if (' ' == pattern[rhs_pos]) rhs_pos++;
}
/* Note: This section is sensitive to the disjunct print format. */
const size_t added_chars_len = sizeof("= <> $");
const size_t regpat_size = 2 * (pat_len + added_chars_len);
regpat = alloca(regpat_size);
size_t dst_pos = lg_strlcpy(regpat, "= ", regpat_size);
if (rhs_pos == 0)
regpat[dst_pos++] = ' '; /* when no LHS, we need "= " */
else
dst_pos += copy_quoted("*+", regpat+dst_pos, pattern, rhs_pos);
if (regpat[dst_pos-1] != ' ') regpat[dst_pos++] = ' ';
dst_pos += lg_strlcpy(regpat + dst_pos, "<> ", regpat_size - dst_pos);
dst_pos += copy_quoted("*+", regpat+dst_pos, pattern+rhs_pos,
pat_len-rhs_pos);
regpat[dst_pos++] = '$';
regpat[dst_pos++] = '\0';
}
else
{
if ((is_regex != NULL) || pattern[strcspn(pattern, "({[.?$\\")] != '\0')
{
regpat = strdupa(pattern);
is_regex = "r";
}
else
{
/* No regex flag and no regex characters (* and + are ignored). */
regpat = alloca(2 * pat_len + 1);
size_t ncopied = copy_quoted("*+", regpat, pattern, pat_len);
regpat[ncopied] = '\0';
}
if (is_anyorder != NULL)
{
/* Assume this is an unordered list of blank-separated connectors. */
char *constring;
while ((constring = strtok_r(regpat, " ", ®pat)))
{
if (is_regex == NULL)
{
/* Arrange for matching only whole connectors. */
const char added_chars[] = " ( |$)";
char *word_boundary_constring =
alloca(strlen(constring) + sizeof(added_chars));
word_boundary_constring[0] = ' ';
strcpy(word_boundary_constring+1, constring);
strcat(word_boundary_constring+1, added_chars+1);
constring = word_boundary_constring;
}
rn = new_disjunct_regex_node(rn, constring);
}
}
}
if (NULL == rn)
rn = new_disjunct_regex_node(NULL, regpat);
if (!compile_regexs(rn, NULL))
return NULL; /* compile_regexs() issues the error message */
return rn;
}
const char do_display_expr; /* a sentinel to request an expression display */
static bool validate_flags(const char *display_type, const char *flags)
{
const char *known_flags;
if (&do_display_expr == display_type)
known_flags = "lm";
else
known_flags = "afmr";
size_t unknown_flag_pos = strspn(flags, known_flags);
if (flags[unknown_flag_pos] != '\0')
{
prt_error("Error: Token display: Unknown flag \"%c\".\n",
flags[unknown_flag_pos]);
if (&do_display_expr == display_type)
{
prt_error("Valid flags for the \"!!word/\" command "
"(show expression):\n"
"l - low level expression details.\n"
"m - macro context.\n");
}
else
{
prt_error("Valid flags for the \"!!word//\" command "
"(show disjuncts):\n"
"a - any connector order.\n"
"f - full disjunct specification.\n"
"m - macro context for connectors.\n"
"r - regex pattern (automatically detected usually).\n");
}
return false;
}
return true;
}
/**
* Display the information about the given word.
* If the word can split, display the information about each part.
* Note that the splits may be invalid grammatically.
*
* Wild-card search is supported; the command-line user can type in !!word* or
* !!word*.sub and get a list of all words that match up to the wild-card.
* In this case no split is done.
*
* @param arg arg[0] specifies the display type. If it is equal to
* &do_display_expr than this is request to display expressions. Else it
* is a request to display disjuncts. arg[2] specifies the request flags.
*/
static char *display_word_split(Dictionary dict,
const char * word, Parse_Options opts,
char * (*display)(Dictionary, const char *, const void **),
const char **arg)
{
if ('\0' == *word) return NULL; /* avoid trying null strings */
/* SUBSCRIPT_DOT in a sentence word is not interpreted as SUBSCRIPT_MARK,
* and hence a subscripted word that is found in the dict will not
* get found in the dict if it can split. E.g: 's.v (the info for s.v
* will not be shown). Fix it by replacing it to SUBSCRIPT_MARK. */
char *pword = strdupa(word);
patch_subscript(pword);
dyn_str *s = dyn_str_new();
int spell_option = parse_options_get_spell_guess(opts);
parse_options_set_spell_guess(opts, 0);
Sentence sent = sentence_create(pword, dict);
if (pword[0] == '<' && (strchr(pword, '>') != NULL) &&
((strchr(pword, '>')[1] == '\0') ||
(strchr(pword, '>')[1] == SUBSCRIPT_MARK)))
{
/* Dictionary macro - don't split. */
if (!word0_set(sent, pword, opts)) goto display_word_split_error;
}
else
{
if (0 != sentence_split(sent, opts)) goto display_word_split_error;
}
/* List the splits */
print_sentence_word_alternatives(s, sent, false, NULL, NULL, NULL);
/* List the expression / disjunct information */
/* Initialize the callback arguments */
const void *carg[3] = { /*regex*/NULL, /*flags*/NULL, opts };
Regex_node *rn = NULL;
if (arg != NULL)
{
if (NULL != arg[1])
{
if (!validate_flags(arg[0], arg[1]))
{
dyn_strcat(s, " "); /* avoid no-expression error */
goto display_word_split_error;
}
}
carg[1] = arg[1]; /* flags */
if (arg[0] == &do_display_expr)
{
carg[0] = &do_display_expr;
}
else if (arg[0] != NULL)
{
/* A pattern is specified, which means displaying disjuncts. */
if (arg[0][0] != '\0')
{
rn = make_disjunct_pattern(arg[0], arg[1]);
if (NULL == rn)
{
dyn_strcat(s, " "); /* avoid a no-match error */
goto display_word_split_error;
}
carg[0] = rn;
}
}
}
print_sentence_word_alternatives(s, sent, false, display, carg, NULL);
if (rn != NULL) free_regexs(rn);
display_word_split_error:
sentence_delete(sent);
parse_options_set_spell_guess(opts, spell_option);
char *out = dyn_str_take(s);
if ('\0' != out[0]) return out;
free(out);
return NULL; /* no dict entry */
}
/**
* Count the number of clauses (disjuncts) for the expression e.
* Should return the number of disjuncts that would be returned
* by build_disjunct(). This in turn should be equal to the number
* of clauses built by build_clause().
*
* Only one minor cheat here: we are ignoring the cost_cutoff, so
* this potentially over-counts if the cost_cutoff is set low.
*/
uint64_t count_clause(const Exp *e)
{
uint64_t cnt;
assert(e != NULL, "count_clause called with null parameter");
if (e->type == AND_type)
{
/* multiplicative combinatorial explosion */
cnt = 1;
for (Exp *opd = e->operand_first; opd != NULL; opd = opd->operand_next)
cnt *= count_clause(opd);
}
else if (e->type == OR_type)
{
/* Just additive */
cnt = 0;
for (Exp *opd = e->operand_first; opd != NULL; opd = opd->operand_next)
cnt += count_clause(opd);
}
else if (e->type == CONNECTOR_type)
{
return 1;
}
else
{
assert(false, "Unknown expression type %d", (int)e->type);
}
return cnt;
}
/**
* Count number of disjuncts given the dict node dn.
*/
static uint64_t count_disjunct_for_dict_node(Dict_node *dn)
{
return (NULL == dn) ? 0 : count_clause(dn->exp);
}
/**
* Display the number of disjuncts associated with this dict node
*/
static char *display_counts(const char *word, Dict_node *dn)
{
dyn_str *s = dyn_str_new();
dyn_strcat(s, "matches:\n");
for (; dn != NULL; dn = dn->right)
{
append_string(s, " %-*s %8"PRIu64" disjuncts",
display_width(DJ_COL_WIDTH, dn->string), dn->string,
count_disjunct_for_dict_node(dn));
if (dn->file != NULL)
{
append_string(s, " <%s>", dn->file);
}
dyn_strcat(s, "\n\n");
}
return dyn_str_take(s);
}
/**
* Display the expressions associated with this dict node.
*/
static char *display_expr(Dictionary dict, const char *word, Dict_node *dn,
const void **arg)
{
const char *flags = arg[1];
const Parse_Options opts = (Parse_Options)arg[2];
bool show_macros = ((flags != NULL) && (strchr(flags, 'm') != NULL));
bool low_level = ((flags != NULL) && (strchr(flags, 'l') != NULL));
/* copy_Exp() needs an Exp memory pool. */
Pool_desc *Exp_pool = pool_new(__func__, "Exp", /*num_elements*/256,
sizeof(Exp), /*zero_out*/false,
/*align*/false, /*exact*/false);
dyn_str *s = dyn_str_new();
dyn_strcat(s, "expressions:\n");
for (; dn != NULL; dn = dn->right)
{
Exp *e = copy_Exp(dn->exp, Exp_pool, opts); /* assign dialect costs */
pool_reuse(Exp_pool);
if (low_level)
{
append_string(s, " %s\n", dn->string);
prt_exp_all(s, e, 0, dict);
dyn_strcat(s, "\n\n");
}
char *expstr = lg_exp_stringify_with_tags(dict, e, show_macros);
append_string(s, " %-*s %s",
display_width(DJ_COL_WIDTH, dn->string), dn->string,
expstr);
dyn_strcat(s, "\n\n");
free(expstr);
}
if (Exp_pool != NULL) pool_delete(Exp_pool);
return dyn_str_take(s);
}
/**
* A callback function to display \p word number of disjuncts and file name.
*
* @param arg Callback args (unused).
* @return String to display. Must be freed by the caller.
*/
static char *display_word_info(Dictionary dict, const char *word,
const void **arg)
{
const char * regex_name;
Dict_node *dn_head;
dn_head = dictionary_lookup_wild(dict, word);
if (dn_head)
{
char *out = display_counts(word, dn_head);
free_lookup_list(dict, dn_head);
return out;
}
/* Recurse, if it's a regex match */
regex_name = match_regex(dict->regex_root, word);
if (regex_name)
{
return display_word_info(dict, regex_name, arg);
}
return NULL;
}
/**
* A callback function to display \p word expressions or disjuncts.
* @param arg Callback data as follows:
* arg[0]: &do_display_expr or disjunct selection regex.
* arg[1]: flags
* arg[2]: Parse_Options
* @return String to display. Must be freed by the caller.
*/
static char *display_word_expr(Dictionary dict, const char *word,
const void **arg)
{
const char * regex_name;
Dict_node *dn_head;
char *out = NULL;
dn_head = dictionary_lookup_wild(dict, word);
if (dn_head)
{
if (arg[0] == &do_display_expr)
{
out = display_expr(dict, word, dn_head, arg);
}
else
{
out = display_disjuncts(dict, dn_head, arg);
}
free_lookup_list(dict, dn_head);
return out;
}
/* Recurse, if it's a regex match */
regex_name = match_regex(dict->regex_root, word);
if (regex_name)
{
return display_word_expr(dict, regex_name, arg);
}
return NULL;
}
static char *find_unescaped_slash(char *word)
{
size_t len = strlen(word);
for (char *src = word, *dst = word; *src != '\0'; src++, dst++)
{
if (('\\' == *src) && (('\\' == src[1]) || ('/' == src[1])))
{
memmove(dst, src+1, len - (src - word));
}
else
{
if ('/' == *src) return dst;
}
}
return NULL;
}
/**
* Break "word", "word/flags" or "word/regex/flags" into components.
* "regex" and "flags" are optional. "word/" means an empty regex.
* \p re and \p flags can be both NULL.
* @param re[out] the regex component, unless \c NULL.
* @param flags[out] the flags component, unless \c NULL.
* @return The word component.
*/
static const char *display_word_extract(char *word, const char **re,
const char **flags)
{
if (re != NULL) *re = NULL;
if (flags != NULL) *flags = NULL;
char *r = find_unescaped_slash(word);
if (r == NULL) return word;
*r = '\0';
if (re != NULL)
{
char *f = find_unescaped_slash(r + 1);
if (f != NULL)
{
*re = r + 1;
*f = '\0';
*flags = f + 1; /* disjunct display flags */
}
else
{
*flags = r + 1; /* expression display flags */
}
}
return word;
}
/**
* dict_display_word_info() - display the information about the given word.
*/
char *dict_display_word_info(Dictionary dict, const char *word,
Parse_Options opts)
{
char *wordbuf = strdupa(word);
word = display_word_extract(wordbuf, NULL, NULL);
if ('\0' == *word)
{
prt_error("Error: Missing word argument.\n");
return strdup(" ");
}
return display_word_split(dict, word, opts, display_word_info, NULL);
}
/**
* dict_display_word_expr() - display the connector info for a given word.
*/
char *dict_display_word_expr(Dictionary dict, const char * word, Parse_Options opts)
{
const char *arg[2];
char *wordbuf = strdupa(word);
word = display_word_extract(wordbuf, &arg[0], &arg[1]);
if ('\0' == *word) return strdup(" ");
/* If no regex component, then it's a request to display expressions. */
if (arg[0] == NULL) arg[0] = &do_display_expr;
return display_word_split(dict, word, opts, display_word_expr, arg);
}