/*************************************************************************/
/* Copyright (c) 2004 */
/* Daniel Sleator, David Temperley, and John Lafferty */
/* All rights reserved */
/* */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software. */
/* This license allows free redistribution and use in source and binary */
/* forms, with or without modification, subject to certain conditions. */
/* */
/*************************************************************************/
#ifndef _TOKENIZE_H
#define _TOKENIZE_H
#include "api-types.h"
#include "link-includes.h"
bool separate_sentence(Sentence, Parse_Options);
bool sentence_in_dictionary(Sentence);
void flatten_wordgraph(Sentence, Parse_Options);
void tokenization_done(Sentence, Gword *);
void free_words(Sentence);
void altappend(Sentence, const char ***, const char *);
bool word0_set(Sentence, char *, Parse_Options);
Gword *issue_word_alternative(Sentence sent, Gword *unsplit_word,
const char *label,
int prefnum, const char * const *prefix,
int stemnum, const char * const *stem,
int suffnum, const char * const *suffix);
#endif /* _TOKENIZE_H */