%***************************************************************************%
% %
% Copyright (C) 2005, 2006 Sampo Pyysalo, Sophie Aubin %
% Copyright (C) 2009, 2012 Linas Vepstas %
% Copyright (C) 2022 Prachya Boonkwan %
% See file "LICENSE" for information about commercial use of this system %
% %
%***************************************************************************%
% This file contains regular expressions that are used to match
% tokens not found in the dictionary. Each regex is given a name which
% determines the disjuncts assigned when the regex matches; this name
% must be defined in the dictionary along with the appropriate disjuncts.
% Note that the order of the regular expressions matters: matches will
% be attempted in the order in which the regexs appear in this file,
% and only the first match will be used.
%
% Regex'es that are preceded by !, if they match a token, stop
% further match tries of the same regex name. Thus, they can serve
% as a kind of a negative look-ahead.
%
% To allow non-ASCII ranges in character classes for libraries other
% than recent PCRE2, such ranges are expanded by default. Since various
% regex libraries have different syntax features for character classes,
% it is hard to do such an expansion only inside character classes, so
% this expansion is always done. To avoid it, add:
% NO-EXPAND: //
% Numbers.
% XXX, we need to add utf8 U+00A0 "no-break space"
%
% Allows at most two colons in hour-minute-second HH:MM:SS expressions
% Allows at most two digits between colons
HMS-TIME: /^[0-9][0-9]?(:[0-9][0-9]?(:[0-9][0-9]?)?)?(AM|PM|am|pm)?$/
% e.g. 1950's leading number can be higher, for science fiction.
% Must be four digits, or possible three. Must end in s, 's ’s
DECADE-DATE: /^([1-4][0-9][0-9]|[1-9][0-9])0(s|'s|’s)$/
% Similar to above, but does not end in s. Only allows four digits.
% We process this before NUMBERS below, so that this is matched first.
YEAR-DATE: /^([1-4][0-9]{3}|[1-9][0-9]{0,2})$/
% Allows any number of commas or periods
% Be careful not match the period at the end of a sentence;
% for example: "It happened in 1942."
NUMBERS: /^[0-9,.]*[0-9]$/
% This parses signed numbers and ranges, e.g. "-5" and "5-10" and "9+/-6.5"
NUMBERS: /^[0-9.,-]*[0-9](\+\/-[0-9.,-]*[0-9])?$/
% Parses simple fractions e.g. "1/60" with no decimal points or anything fancy
FRACTION: /^[0-9]+\/[0-9]+$/
% "10(3)" exponent (used in PubMed)
NUMBERS: /^[0-9.,-]*[0-9][0-9.,-]*\([0-9:.,-]*[0-9][0-9.,-]*\)$/
% Thai numbers
THAI-NUMBERS: /^[๐-๙,.]*[๐-๙]$/
THAI-NUMBERS: /^[๐-๙.,-]*[๐-๙](\+\/-[๐-๙.,-]*[๐-๙])?$/
THAI-FRACTION: /^[๐-๙]+\/[๐-๙]+$/
THAI-NUMBERS: /^(หนึ่ง|สอง|สาม|สี่|ห้า|หก|เจ็ด|แปด|เก้า|สิบ|ร้อย|พัน|หมื่น|แสน|ล้าน|เอ็ด|ศูนย์|ยี่สิบ)+$/
THAI-NUMBERS: /^[๐-๙.,-]*[๐-๙][๐-๙.,-]*\([๐-๙:.,-]*[๐-๙][๐-๙.,-]*\)$/
% Thai Date and Time
THAI-DATE: /^(วัน)?_?(จันทร์|อังคาร|พุธ|พฤหัสบดี|ศุกร์|เสาร์|อาทิตย์|พฤหัส)?_?(ที่)?_?[0-9๐-๙]+_?(มกราคม|กุมภาพันธ์|มีนาคม|เมษายน|พฤษภาคม|มิถุนายน|กรกฎาคม|สิงหาคม|กันยายน|ตุลาคม|พฤศจิกายน|ธันวาคม|(มกรา|กุมภา|มีนา|เมษา|พฤษภา|มิถุนา|กรกฎา|สิงหา|กันยา|ตุลา|พฤศจิกา|ธันวา)ฯ?|ม\.ค\.|ก\.พ\.|มี\.ค\.|เม\.ย\.|พ\.ค\.|มิ\.ย\.|ก\.ค\.|ส\.ค\.|ก\.ย\.|ต\.ค\.|พ\.ย\.|ธ\.ค\.)_?(พ\.ศ\.|ค\.ศ\.)?_?[0-9๐-๙]+$/
THAI-TIME: /^(เวลา)?_?([0-9๐-๙]+(\.[0-9๐-๙]+)?_?น\.|[0-9๐-๙]+_?(นาฬิกา|โมง)_?(ตรง|_?[0-9๐-๙]+_?นาที)?_?([0-9๐-๙]+_?วินาที)?)$/
% Thai colors
THAI-COLOR: /^(สี)?_?(เหลือบ)?_?(ดำ|ขาว|เหลือง|แดง|น้ำเงิน|ส้ม|เขียว|ม่วง|ชมพู|ฟ้า|แสด|ทอง|เงิน|ทองแดง|บรอนซ์|บลอนด์|โอลด์โรส|น้ำตาล|น้ำ_?ทะเล|คราม|กรมท่า|เลือด_?นก|เลือด_?หมู|บานเย็น|น้ำตาล_?ไหม้|กาแฟ|โกโก้|เทา|ครีม|ไข่มุก)_?(เข้ม|อ่อน|เรื่อ_?ๆ?|ซีด_?ๆ?|(ปน_?)?(ดำ|ขาว|เหลือง|แดง|น้ำเงิน|ส้ม|เขียว|ม่วง|ชมพู|ฟ้า|แสด|ทอง|เงิน|ทองแดง|บรอนซ์|บลอนด์|โอลด์โรส|น้ำตาล|น้ำ_?ทะเล|คราม|กรมท่า|เลือด_?นก|เลือด_?หมู|บานเย็น|น้ำตาล_?ไหม้|กาแฟ|โกโก้|เทา|ครีม|ไข่มุก))?$/
% Roman numerals
% The first expr has the problem that it matches an empty string. The
% cure for this is to use look-ahead, but neither the Gnu nor the BSD
% regex libs support look-ahead. I can't think of a better solution.
ROMAN-NUMERAL-WORDS: /^M*(CM|D?C{0,3}|CD)(XC|L?X{0,3}|XL)(IX|V?I{0,3}|IV)$/
% Other proper nouns.
% We demand that these end with an alphanumeric, i.e. explicitly
% reject punctuation. We don't want this regex to "swallow" any trailing
% commas, colons, or periods/question-marks at the end of sentences.
% In addition, this must not swallow words ending in 's 'll etc.
% (... any affix, for that matter ...) and so no embedded apostrophe
CAPITALIZED-WORD: /^[[:upper:]][^'’]*[^[:punct:]]$/
% Hyphenated words. In the original LG morpho-guessing system that
% predated the regex-based system, hyphenated words were detected
% before ING-WORDS, S-WORDS etc., causing e.g. "cross-linked" to be
% treated as a HYPHENATED-WORD (a generic adjective/noun), and
% never a verb. To return to this ordering, move this regex just
% after the CAPITALIZED-WORDS regex.
% We also match on commas, dots, brackets: n-amino-3-azabicyclo[3.3.0]octane
% []...] means "match right-bracket" -> Replaced by |\] because it
% doesn't work in c++ <regex>.
% Explicitly call out (5'|3') so that we don't all a generic match to 'll
% But something is funky about this 5'-3' business since 2' also matches ???
% /^[[:alnum:]][][:alnum:],:.\[-]*-[][:alnum:],:.\[-]*[[:alnum:]]$/
HYPHENATED-WORD:
/^([[:alnum:]]|5'|3'|2'|N')([[:alnum:],:.()[-]|\]|5'|3'|2'|N')*-([[:alnum:],:.()[-]|\])*[[:alnum:]]*$/
% Emoticon checks must come *after* the above, so that the above take precedence.
% See Wikipedia List_of_emoticons (also the References section).
%
% Emoticons must be entirely made of punctuation, length 2 or longer ;)
% XXX [:punct:] is strangely broken, I have to add ;-< explicitly
% XXX: Don't use [:punct:]. Do NOT include period!!
% XXX: The problem with below is that 5. 7. 8. get recognized as emoticons,
% which then prevents splitting for list numbers. (e.g "step 5. Do this.")
%
% Arghh. Other valid number expressions are clobbered by the emoticons.
% For example: $5 $7 8% The quick fix is to remove the numbers.
% Other breakages: The below clobbers "Bob, who ..." because it
% matches Bob, as an emoticon.
%
% EMOTICON: /^[[:punct:];BDOpTX0578Ｃ☆ಠ●＠◎～][[:punct:]<bcdDLmoOpPSTvX0358ಠっ○ 。゜✿☆＊レツ◕●≧∇≦□◇＠◎∩ω旦ヨ彡ミ‿◠￣ー～━-]+$/
% EMOTICON: /^[!"#$%&'()*+,\-/:;<=>?@[\\\]^_`{|}~;BDOpTX0578Ｃ☆ಠ●＠◎～][!"#$%&'()*+,\-/:;<=>?@[\\\]^_`{|}~<bcdDLmoOpPSTvX0358ಠっ○ 。゜✿☆＊レツ◕●≧∇≦□◇＠◎∩ω旦ヨ彡ミ‿◠￣ー～━-]+$/
EMOTICON: !/^"|[[:alnum:]]+"$/
EMOTICON: /^[[:punct:];BＣ☆ಠ●＠◎～][-!"#$%&'()+,:;<=>?@[\\^_`{|}~<cdDLmoOpPSTvXಠっ○ 。゜✿☆＊レツ◕●≧∇≦□◇＠◎∩ω旦ヨ彡ミ‿◠￣ー～━-]+$/
% Part numbers should not match words with punctuation at their end.
% Else sentences like "I saw him on January 21, 1990" have problems.
% They should contain at least one number, and should not have dashes at their
% start or end. A $ sign at the start is also too confusing.
% The current regex system and the syntax of this file are not expressive enough
% for things that should not be included. For example, we cannot prevent several
% sequential "#" or dashes. It may match a word consisting of number+units, but
% separate_word() will generate an alternative anyway.
% The second part of this regex is for NNN-NNN in sentences like
% "The plane is a 747-400". However, such words currently match NUMBERS.
PART-NUMBER: /^[A-Z0-9#][A-Z0-9$\/#]*[A-Z0-9$\/#,.-]*[0-9][A-Z0-9$\/#,.-]*[A-Z0-9$\/#]+$|^[1-9][0-9]+[\/-][0-9]+$/
THAI-PART-NUMBER: /^[A-Zก-ฮ0-9๐-๙#][A-Zก-ฮ0-9๐-๙$\/#]*[A-Zก-ฮ0-9๐-๙$\/#,.-]*[0-9๐-๙][A-Zก-ฮ0-9๐-๙$\/#,.-]*[A-Zก-ฮ0-9๐-๙$\/#]+$|^[1-9๑-๙][0-9๐-๙]+[\/-][0-9๐-๙]+$/
% Sequence of punctuation marks. If some mark appears in the affix table
% such as a period, comma, dash or underscore, and there's a sequence of
% these, then treat it as a "fill-in-the-blank" placeholder.
% This matters only for punc. appearing in the affix table, since the
% tokenizer explicitly mangles based on these punctuation marks.
%
% Look for at least four in a row.
% UNKNOWN-WORD: /^[.,-]{4}[.,-]*$/
ENGLISH-WORD: /^[A-Za-z'’]+$/
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% Words Annotated with POS tags
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Link Grammar's POS Tag
W-ADJ: /\.j$/
W-ADJ-LHS: /\.jl$/
W-ADV: /\.r$/
W-ADV-ATTRMOD: /\.ra$/
W-ADV-COHESIVE: /\.rc$/
W-ADV-QF: /\.rq$/
W-AUX: /\.x$/
W-CL-NOUN: /\.cln$/
W-CL-VERB: /\.clv$/
W-CONJ-DISCONT: /\.cd$/
W-CONJ-NOUN: /\.cn$/
W-CONJ-PREP: /\.cp$/
W-CONJ-VERB: /\.cv$/
W-INTJ: /\.ij$/
W-NOUN-ATTR: /\.na$/
W-NOUN: /\.n$/
W-NOUN-TITLE: /\.nt$/
W-NOUN-PREFIX: /\.fx$/
W-NEG-LEFT: /\.ngl$/
W-NEG-RIGHT: /\.ngr$/
W-NUM: /\.nu$/
W-NUM-MOV: /\.nuv$/
W-ORDMARK: /\.om$/
W-PARTICLE: /\.pt$/
W-PASSMARK: /\.ps$/
W-PREP: /\.p$/
W-PREP-NN: /\.pnn$/
W-PREP-VN: /\.pvn$/
W-PREP-AN: /\.pan$/
W-PREP-NV: /\.pnv$/
W-PREP-VV: /\.pvv$/
W-PREP-AV: /\.pav$/
W-PREP-NA: /\.pna$/
W-PREP-VA: /\.pva$/
W-PREP-AA: /\.paa$/
W-PREP-LW: /\.pw$/
W-PRO: /\.pr$/
W-QF-LEFT: /\.qfl$/
W-QF-RIGHT: /\.qfr$/
W-RELPRO: /\.rl$/
W-SCLMARK: /\.sm$/
W-VERB: /\.v$/
W-VERB-ATTR: /\.va$/
W-VERB-GER: /\.vg$/
W-VERB-EVIDENTIAL: /\.ve$/
% Mapping to LST20 POS Tags
W-LST20-AJ: /\@AJ$/
W-LST20-AV: /\@AV$/
W-LST20-AX: /\@AX$/
W-LST20-CC: /\@CC$/
W-LST20-CL: /\@CL$/
W-LST20-FX: /\@FX$/
W-LST20-IJ: /\@IJ$/
W-LST20-NG: /\@NG$/
W-LST20-NN: /\@NN$/
W-LST20-NU: /\@NU$/
W-LST20-PA: /\@PA$/
W-LST20-PR: /\@PR$/
W-LST20-PS: /\@PS$/
W-LST20-PU: /\@PU$/
W-LST20-VV: /\@VV$/
W-LST20-XX: /\@XX$/
% Named Entities
NE-LST20-PER : /\@PER$/ % Person
NE-LST20-TTL : /\@TTL$/ % Title
NE-LST20-DES : /\@DES$/ % Designator
NE-LST20-ORG : /\@ORG$/ % Organization
NE-LST20-LOC : /\@LOC$/ % Location
NE-LST20-BRN : /\@BRN$/ % Brand
NE-LST20-DTM : /\@DTM$/ % Date and time
NE-LST20-MEA : /\@MEA$/ % Measurement unit
NE-LST20-NUM : /\@NUM$/ % Number
NE-LST20-TRM : /\@TRM$/ % Terminology