/*
* read-sql.c
*
* Look up words in an SQL DB.
* Keeping it simple for just right now, and using SQLite3.
*
* The goal of using an SQL-backed dictionary is to enable some
* other process (machine-learning algo) to dynamically update
* the dictionary.
*
* Copyright (c) 2014,2021 Linas Vepstas <linasvepstas@gmail.com>
*/
#ifdef HAVE_SQLITE3
#define D_SQL 5 /* Verbosity levels for this module are 5 and 6. */
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#if HAVE_THREADS_H
#include <threads.h> // for mtx_t
#endif
#include <sqlite3.h>
#include "api-structures.h"
#include "connectors.h"
#include "dict-common/dict-affix-impl.h"
#include "dict-common/dict-api.h"
#include "dict-common/dict-common.h"
#include "dict-common/dict-internals.h"
#include "dict-common/dict-locale.h"
#include "dict-common/dict-structures.h"
#include "dict-common/dict-utils.h" // patch_subscript()
#include "dict-common/file-utils.h"
#include "dict-file/read-dict.h" // dictionary_six()
#include "error.h"
#include "externs.h"
#include "memory-pool.h"
#include "string-set.h"
#include "tokenize/spellcheck.h"
#include "utilities.h"
#include "read-sql.h"
/* ========================================================= */
/* Mini expression-parsing library. This is a simplified subset of
* what can be found in the file-backed dictionary.
*
* This does NOT support braces {} used to indicate optional connectors,
* so using this in the SQL tables is NOT valid! It also does not
* support cost brackets []. Costs are handled out-of-line.
*
* This is not meant to be "just like the text files, but different".
*/
static const char * make_expression(Dictionary dict,
const char *exp_str, Exp** pex)
{
*pex = NULL;
/* Search for the start of a connector */
Exp_type etype = CONNECTOR_type;
const char * p = exp_str;
while (*p && (lg_isspace((unsigned char)*p))) p++;
if (0 == *p) return p;
/* If it's an open paren, assume it's the beginning of a new list */
if ('(' == *p)
{
p = make_expression(dict, ++p, pex);
}
else
{
/* Search for the end of a connector */
const char * con_start = p;
while (*p && (isalnum((unsigned char)*p) || '*' == *p)) p++;
/* Connectors always end with a + or - */
assert (('+' == *p) || ('-' == *p),
"Missing direction character in connector string: %s", con_start);
/* Create an expression to hold the connector */
char * constr = NULL;
bool multi = false;
if ('@' == *con_start)
{
constr = strndupa(con_start+1, p-con_start-1);
multi = true;
}
else
constr = strndupa(con_start, p-con_start);
Exp* e = make_connector_node(dict, dict->Exp_pool, constr, *p, multi);
*pex = e;
}
/* Is there any more? If not, return what we've got. */
p++;
while (*p && (lg_isspace((unsigned char)*p))) p++;
if (')' == *p || 0 == *p)
{
return p;
}
/* Wait .. there's more! */
if ('&' == *p)
{
etype = AND_type; p++;
}
else if ('o' == *p && 'r' == *(p+1))
{
etype = OR_type; p+=2;
}
else
{
assert(false, "Bad rest of expression %s", exp_str);
}
Exp* rest = NULL;
p = make_expression(dict, p, &rest);
assert(NULL != rest, "Badly formed expression %s", exp_str);
/* Join it all together. */
Exp* join = make_join_node(dict->Exp_pool, *pex, rest, etype);
*pex = join;
return p;
}
/* ========================================================= */
/* Dictionary word lookup procedures. */
#if HAVE_THREADS_H
static mtx_t global_mutex;
#endif
typedef struct
{
Dictionary dict;
Dict_node* dn;
bool found;
int count;
Exp* exp;
char* classname;
} cbdata;
/* callback -- set bs->exp to the expressions for a class in the dict */
static int exp_cb(void *user_data, int argc, char **argv, char **colName)
{
cbdata* bs = user_data;
Dictionary dict = bs->dict;
assert(2 == argc, "Bad column count");
assert(argv[0], "NULL column value");
Exp* exp = NULL;
make_expression(dict, argv[0], &exp);
assert(NULL != exp, "Failed expression %s", argv[0]);
if (!strtofC(argv[1], &exp->cost))
{
prt_error("Warning: Invalid cost \"%s\" in expression \"%s\" "
"(using 1.0)\n", argv[1], argv[0]);
exp->cost = 1.0;
}
/* If the very first expression, just put it in place */
if (NULL == bs->exp)
{
bs->exp = exp;
return 0;
}
/* If the second expression, OR-it with the existing expression. */
if (OR_type != bs->exp->type)
{
bs->exp = make_or_node(dict->Exp_pool, exp, bs->exp);
return 0;
}
/* Extend the OR-chain for the third and later expressions. */
exp->operand_next = bs->exp->operand_first;
bs->exp->operand_first = exp;
return 0;
}
/* Escape single-quotes. That is, replace single-quotes by
* two single-quotes. e.g. don't --> don''t */
static char * escape_quotes(const char * s)
{
char * q = strchr(s, '\'');
if (NULL == q) return (char *) s;
/* If there are two in a row, already, assume that they are
* already escaped. This is pathological, and a sign that
* something somewhere else has already gone wrong. But I'm
* exhausted debugging right now and will punt on this.
*/
if ('\'' == *(q+1)) return (char *) s;
char * es = malloc(2 * strlen(s) + 1);
char * p = es;
while (q)
{
strncpy(p, s, q-s+1);
p += q-s+1;
*p = '\'';
p++;
s = q+1;
q = strchr(s, '\'');
}
strcpy(p, s);
return es;
}
static void
db_lookup_exp(Dictionary dict, const char *s, cbdata* bs)
{
#if HAVE_THREADS_H
mtx_lock(&global_mutex);
#endif
sqlite3 *db = dict->db_handle;
dyn_str *qry;
/* Escape single-quotes. That is, replace single-quotes by
* two single-quotes. e.g. don't --> don''t */
char * es = escape_quotes(s);
/* The token to look up is called the 'morpheme'. */
qry = dyn_str_new();
dyn_strcat(qry, "SELECT disjunct, cost FROM Disjuncts WHERE classname = \'");
dyn_strcat(qry, es);
dyn_strcat(qry, "\';");
sqlite3_exec(db, qry->str, exp_cb, bs, NULL);
dyn_str_delete(qry);
if (es != s) free(es);
lgdebug(D_SQL+1, "Found expression for class %s: %s\n",
s, exp_stringify(bs->exp));
#if HAVE_THREADS_H
mtx_unlock(&global_mutex);
#endif
}
/* callback -- set bs->found to true if the word is in the dict */
static int exists_cb(void *user_data, int argc, char **argv, char **colName)
{
cbdata* bs = user_data;
assert(2 == argc, "Bad column count");
assert(argv[0], "NULL column value");
bs->found = true;
return 0;
}
/* callback -- set bs->dn to the dict nodes for a word in the dict */
static int morph_cb(void *user_data, int argc, char **argv, char **colName)
{
assert(2 == argc, "Bad column count");
assert(argv[0], "NULL column value");
char * scriword = argv[0];
char * wclass = argv[1];
/* Now look up the expressions for each word */
cbdata* bs = user_data;
bs->exp = NULL;
db_lookup_exp(bs->dict, wclass, bs);
/* Well, if we found a classname for a word, then there really,
* really should be able to find one or more corresponding disjuncts.
* However, it is possible to have corrupted databases which do not
* have any disjuncts for a word class. We complain about those.
*/
assert(NULL != bs->exp, "Missing disjuncts for word %s %s",
scriword, wclass);
#if HAVE_THREADS_H
mtx_lock(&global_mutex);
#endif
/* Put each word into a Dict_node. */
Dict_node *dn = dict_node_new();
dn->string = string_set_add(scriword, bs->dict->string_set);
dn->right = bs->dn;
dn->exp = bs->exp;
bs->dn = dn;
#if HAVE_THREADS_H
mtx_unlock(&global_mutex);
#endif
return 0;
}
static void
db_lookup_common(Dictionary dict, const char *s, const char *equals,
int (*cb)(void *, int, char **, char **),
cbdata* bs)
{
sqlite3 *db = dict->db_handle;
dyn_str *qry;
/* Escape single-quotes. That is, replace single-quotes by
* two single-quotes. e.g. don't --> don''t */
char * es = escape_quotes(s);
/* The token to look up is called the 'morpheme'. */
qry = dyn_str_new();
dyn_strcat(qry, "SELECT subscript, classname FROM Morphemes WHERE morpheme ");
dyn_strcat(qry, equals);
dyn_strcat(qry, " \'");
dyn_strcat(qry, es);
dyn_strcat(qry, "\';");
if (es != s) free(es);
sqlite3_exec(db, qry->str, cb, bs, NULL);
dyn_str_delete(qry);
}
static bool db_lookup(Dictionary dict, const char *s)
{
cbdata bs;
bs.dict = dict;
bs.found = false;
db_lookup_common(dict, s, "=", exists_cb, &bs);
return bs.found;
}
static Dict_node * db_lookup_list(Dictionary dict, const char *s)
{
cbdata bs;
bs.dict = dict;
bs.dn = NULL;
db_lookup_common(dict, s, "=", morph_cb, &bs);
if (verbosity_level(D_SQL))
{
if (bs.dn)
{
printf("Found expression for word %s: %s\n",
s, exp_stringify(bs.dn->exp));
}
else
{
printf("No expression for word %s\n", s);
}
}
return bs.dn;
}
/**
* This is used to support wild-card lookup from the command-line
* client. That is, a user can type in `!!foo*` and look up all
* words that begin with the three letters `foo`. It works ...
* but it only works if the dictionary also has UNKNOWN-WORD defined!
*/
static Dict_node * db_lookup_wild(Dictionary dict, const char *s)
{
cbdata bs;
bs.dict = dict;
bs.dn = NULL;
db_lookup_common(dict, s, "GLOB", morph_cb, &bs);
if (verbosity_level(D_SQL))
{
if (bs.dn)
{
printf("Found expression for glob %s: %s\n",
s, exp_stringify(bs.dn->exp));
}
else
{
printf("No expression for glob %s\n", s);
}
}
return bs.dn;
}
/* ========================================================= */
/* Callbacks and functions to support lexical category loading. */
/* Used for `SELECT count(*) FROM foo` type of queries */
static int count_cb(void *user_data, int argc, char **argv, char **colName)
{
cbdata* bs = user_data;
assert(1 == argc, "Bad column count");
bs->count = atol(argv[0]);
return 0;
}
/* Record the name of each lexical class */
static int classname_cb(void *user_data, int argc, char **argv, char **colName)
{
cbdata* bs = user_data;
Dictionary dict = bs->dict;
/* Assuming here that a class name of a wall is the same as the wall
* name, and a class name of a macro is in macro format. */
if (!dict->generate_walls && is_wall(argv[0])) return 0;
if (is_macro(argv[0])) return 0;
/* Add a category. */
/* This is intentionally off-by-one, per design. */
dict->num_categories++;
dict->category[dict->num_categories].num_words = 0;
dict->category[dict->num_categories].word = NULL;
char* esc = escape_quotes(argv[0]);
dict->category[dict->num_categories].name =
string_set_add(esc, dict->string_set);
if (esc != argv[0]) free(esc);
char category_string[16]; /* For the tokenizer - not used here */
snprintf(category_string, sizeof(category_string), " %x",
dict->num_categories); /* ' ': See comment in build_disjuncts() */
string_set_add(category_string, dict->string_set);
return 0;
}
/* Record the words in each lexical class */
static int classword_cb(void *user_data, int argc, char **argv, char **colName)
{
cbdata* bs = user_data;
Dictionary dict = bs->dict;
char *word = strdupa(argv[0]);
patch_subscript(word);
/* Add the word. */
dict->category[dict->num_categories].word[bs->count] =
string_set_add(word, dict->string_set);
bs->count++;
return 0;
}
/* The current design for generation requires that all word categories
* be loaded into RAM before generation starts. This is required because
* a wild-card appearing in the generator forces a loop over all
* categories, so we may as well have them instantly available.
*/
static void db_add_categories(Dictionary dict)
{
sqlite3 *db = dict->db_handle;
cbdata bs;
bs.dict = dict;
#if HAVE_THREADS_H
mtx_lock(&global_mutex);
#endif
/* How many lexical categories are there? Find out. */
sqlite3_exec(db, "SELECT count(DISTINCT classname) FROM Disjuncts;",
count_cb, &bs, NULL);
dict->num_categories = 0;
dict->num_categories_alloced = 1 + bs.count + 1; // skip slot 0 + terminator
dict->category = malloc(dict->num_categories_alloced *
sizeof(*dict->category));
sqlite3_exec(db, "SELECT DISTINCT classname FROM Disjuncts;",
classname_cb, &bs, NULL);
/* Category 0 is unused, intentionally. Not sure why. */
unsigned int ncat = dict->num_categories;
for (unsigned int i=1; i<=ncat; i++)
{
/* For each category, get the expression. */
dyn_str *qry = dyn_str_new();
dyn_strcat(qry,
"SELECT disjunct, cost FROM Disjuncts WHERE classname = \'");
dyn_strcat(qry, dict->category[i].name);
dyn_strcat(qry, "\';");
bs.exp = NULL;
sqlite3_exec(db, qry->str, exp_cb, &bs, NULL);
dyn_str_delete(qry);
dict->category[i].exp = bs.exp;
/* ------------------ */
/* For each category, get the number of words in the category */
qry = dyn_str_new();
dyn_strcat(qry,
"SELECT count(*) FROM Morphemes WHERE classname = \'");
dyn_strcat(qry, dict->category[i].name);
dyn_strcat(qry, "\';");
sqlite3_exec(db, qry->str, count_cb, &bs, NULL);
dyn_str_delete(qry);
dict->category[i].num_words = bs.count;
dict->category[i].word =
malloc(bs.count * sizeof(*dict->category[0].word));
/* ------------------ */
/* For each category, get the (subscripted) words in the category */
qry = dyn_str_new();
dyn_strcat(qry,
"SELECT subscript FROM Morphemes WHERE classname = \'");
dyn_strcat(qry, dict->category[i].name);
dyn_strcat(qry, "\';");
dict->num_categories = i;
bs.count = 0;
sqlite3_exec(db, qry->str, classword_cb, &bs, NULL);
dyn_str_delete(qry);
}
/* Set the termination entry. */
dict->category[dict->num_categories + 1].num_words = 0;
#if HAVE_THREADS_H
mtx_unlock(&global_mutex);
#endif
}
/* ========================================================= */
/* Dictionary creation, setup, open procedures */
static void* db_open(const char * fullname, const void * user_data)
{
#if HAVE_THREADS_H
mtx_init(&global_mutex, mtx_plain);
#endif
/* Is there a file here that can be read? */
FILE * fh = fopen(fullname, "r");
if (NULL == fh)
return NULL;
/* Get the file size, in bytes. */
/* SQLite has a habit of leaving zero-length DB's lying around */
struct stat buf;
int fd = fileno(fh);
fstat(fd, &buf);
fclose(fh);
if (0 == buf.st_size)
return NULL;
/* Found a file, of non-zero length. See if that works. */
sqlite3 *db;
if (sqlite3_open(fullname, &db))
{
prt_error("Error: Can't open database %s: %s\n",
fullname, sqlite3_errmsg(db));
sqlite3_close(db);
return NULL;
}
return (void *) db;
}
static void db_close(Dictionary dict)
{
sqlite3 *db = dict->db_handle;
if (db)
sqlite3_close(db);
dict->db_handle = NULL;
}
static void db_start_lookup(Dictionary dict, Sentence sent)
{
}
static void db_end_lookup(Dictionary dict, Sentence sent)
{
#if HAVE_THREADS_H
mtx_lock(&global_mutex);
#endif
condesc_setup(dict);
#if HAVE_THREADS_H
mtx_unlock(&global_mutex);
#endif
}
Dictionary dictionary_create_from_db(const char *lang)
{
char *dbname;
const char * t;
Dictionary dict;
dict = (Dictionary) malloc(sizeof(struct Dictionary_s));
memset(dict, 0, sizeof(struct Dictionary_s));
/* Language and file-name stuff */
dict->string_set = string_set_create();
t = strrchr (lang, '/');
t = (NULL == t) ? lang : t+1;
dict->lang = string_set_add(t, dict->string_set);
lgdebug(D_USER_FILES, "Debug: Language: %s\n", dict->lang);
#if 0 /* FIXME: Spell checking should be done according to the dict locale. */
/* To disable spell-checking, just set the checker to NULL */
dict->spell_checker = spellcheck_create(dict->lang);
#endif
#if defined HAVE_HUNSPELL || defined HAVE_ASPELL
/* FIXME: Move to spellcheck-*.c */
if (verbosity_level(D_USER_BASIC) && (NULL == dict->spell_checker))
prt_error("Info: %s: Spell checker disabled.\n", dict->lang);
#endif
dict->base_knowledge = NULL;
dict->hpsg_knowledge = NULL;
dbname = join_path (lang, "dict.db");
dict->name = string_set_add(dbname, dict->string_set);
free(dbname);
/* Set up the database */
dict->db_handle = object_open(dict->name, db_open, NULL);
dict->lookup_list = db_lookup_list;
dict->lookup_wild = db_lookup_wild;
dict->free_lookup = dict_node_free_lookup;
dict->exists_lookup = db_lookup;
dict->start_lookup = db_start_lookup;
dict->end_lookup = db_end_lookup;
dict->clear_cache = dict_node_noop;
dict->close = db_close;
dict->dynamic_lookup = true;
condesc_init(dict, 1<<8);
dict->dfine.set = string_id_create();
dict->Exp_pool = pool_new(__func__, "Exp", /*num_elements*/4096,
sizeof(Exp), /*zero_out*/false,
/*align*/false, /*exact*/false);
/* Setup the affix table */
char *affix_name = join_path (lang, "4.0.affix");
dict->affix_table = dictionary_six(lang, affix_name, NULL, NULL, NULL, NULL);
if (dict->affix_table == NULL)
{
prt_error("Error: Could not open affix file %s\n", affix_name);
free(affix_name);
goto failure;
}
free(affix_name);
if (!afdict_init(dict))
goto failure;
if (!dictionary_setup_defines(dict))
goto failure;
/* Initialize word categories, for text generation. */
if (dictionary_generation_request(dict))
db_add_categories(dict);
return dict;
failure:
dictionary_delete(dict);
return NULL;
}
#endif /* HAVE_SQLITE3 */