version=0.2
[RULE-ORDER]
ABBREVIATION-KNOWN PREFIX NUMBER-ORDINAL URL URL-WWW URL-DOMAIN
E-MAIL WORD-PARPREFIX WORD-PARSUFFIX WORD-COMPOUND
ABBREVIATION INITIAL SMILEY REVERSE-SMILEY PUNCTUATION-MULTI DATE-REVERSE DATE
NUMBER-YEAR TIME FRACNUMBER NUMBER CURRENCY WORD-TOKEN WORD PUNCTUATION UNKNOWN

[META-RULES]
SPLITTER=%
NUMBER-ORDINAL = (?>(?:[\.,]?\p{N}+)+)-?(?i)(?: %ORDINALS% )(?:\Z|\P{Lu}|\P{Ll})$
ABBREVIATION-KNOWN = (?:\p{P}*)?(?:\A|[^\p{L}\.])((?:%ABBREVIATIONS%)\.)(?:\Z|\P{L})
WORD-TOKEN =(%TOKENS%)(?:\p{P}*)?$
PREFIX = (?:\A|[^\p{Lu}\.]|[^\p{Ll}\.])(%PREFIXES% )(\p{L}+)
#SUFFIX = ((?:\p{L})+)( %SUFFIXES% )(?:\Z|\P{L})

[RULES]
%include url
%include e-mail
%include smiley

#Ex: (dis)information
WORD-PARPREFIX=(?:\p{Ps}\p{L}+[\p{Pc}\p{Pd}]?\p{Pe}[\p{Pc}\p{Pd}]?)\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)*

#Ex: understand(s)
WORD-PARSUFFIX=\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)*(?:[\p{Pc}\p{Pd}]?\p{Ps}[\p{Pc}\p{Pd}]?\p{L}+\p{Pe})

#Keep dash/underscore connected parts (even if they are in parenthesis)
WORD-COMPOUND=\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)+

#Abbreviations with multiple periods
ABBREVIATION=^(\p{L}{1,3}(?:\.\p{L}{1,3})+\.?)(?:\Z|[,:;])

#retain initials
INITIAL=^(?:\p{Lt}|\p{Lu})\.$

#Homogeneous punctuation (ellipsis etc)
PUNCTUATION-MULTI=(?:\.|\-|[!\?]){2,}

#Date
DATE=\p{N}{1,2}[/\-]\p{N}{1,2}[/-]\p{N}{2,4}
DATE-REVERSE=\p{N}{4}[/\-]\p{N}{1,2}[/\-]\p{N}{1,2}

FRACNUMBER=\p{N}+(?:/\p{N}+)+

NUMBER-YEAR=(['`’‘´]\p{N}{2})(?:\P{N}|\z)

#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:am|pm)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+

CURRENCY=\p{Sc}

WORD=[\p{L}\p{Mn}]+

PUNCTUATION=\p{P}

UNKNOWN=.

[PREFIXES]
\[lL]['`’‘´]
\[dD]['`’‘´]
\[mM]['`’‘´]
\[tT]['`’‘´]
\[sS]['`’‘´]
\[cC]['`’‘´]
\[dD]ell['`’‘´]
DELL['`’‘´]
\[dD]all['`’‘´]
DALL['`’‘´]
\[nN]ell['`’‘´]
NELL['`’‘´]
\[aA]ll['`’‘´]
ALL['`’‘´]
\[sS]ull['`’‘´]
SULL['`’‘´]
\[qQ]uell['`’‘´]
QUELL['`’‘´]
\[qQ]uest['`’‘´]
QUEST['`’‘´]
\[eE]['`’‘´]
\[uU][nN]['`’‘´]
\[sS]enz['`’‘´]
SENZ['`’‘´]
\[cC]om['`’‘´]
COM['`’‘´]
\[cC]os['`’‘´]
COS['`’‘´]
\[aA]nch['`’‘´]
ANCH['`’‘´]
\[dD]ev['`’‘´]
DEV['`’‘´]
\[dD]ov['`’‘´]
DOV['`’‘´]

[SUFFIXES]

[ORDINALS]
o|O
a|A

[TOKENS]
\[lL]['`’‘´]
\[dD]['`’‘´]
\[mM]['`’‘´]
\[tT]['`’‘´]
\[sS]['`’‘´]
cC['`’‘´]
\[dD]ell['`’‘´]
DELL['`’‘´]
\[dD]all['`’‘´]
DALL['`’‘´]
\[nN]ell['`’‘´]
NELL['`’‘´]
\[aA]ll['`’‘´]
ALL['`’‘´]
\[sS]ull['`’‘´]
SULL['`’‘´]
\[qQ]uell['`’‘´]
QUELL['`’‘´]
\[qQ]uest['`’‘´]
QUEST['`’‘´]
\[eE]['`’‘´]
\[uU][nN]['`’‘´]
\[sS]enz['`’‘´]
SENZ['`’‘´]
\[cC]om['`’‘´]
COM['`’‘´]
\[cC]os['`’‘´]
COS['`’‘´]
\[aA]nch['`’‘´]
ANCH['`’‘´]
\[dD]ev['`’‘´]
DEV['`’‘´]
\[dD]i['`’‘´]
DI['`’‘´]

[UNITS]
km
m
cm
mm
g
kg
C
l
s
sec
min
gb
mb
kb


[CURRENCY]
EUR|eur

[ABBREVIATIONS]
ab
abbigl
abbrev
abl
ac
acc
accorc
accr
adatt
aeron
aerodin
affl
agg
agr
agric
alch
alg
alim
allev
allus
alt
anat
ant
antifr
antrop
ar
arald
arc
arch
archeol
aritm
arred
artig
artigl
artt
assic
assol
astr
astrol
astron
att
attrav
aus
autom
avv
avvers
bal
bibl
bioch
biol
ca
cal
cap
card
caus
centr
cfr
chim
chir
cin
class
cod
civ
coll
com
comm
comp
compar
comp
conc
concl
condiz
confr
cong
congiunt
coniug
consec
contab
contr
corr
correl
corrisp
costr
cuc
d
dat
decoraz
denom
deriv
determ
dial
difett
dimin
dimostr
dipart
dir
Dir
Dott
Dr
ebr
acc
accl
acon
ed
edit
elab
elettr
es
escl
estens
etim
fam
fig
fr
g
gen
geol
gr
gram
ibid
id
ig
imp
imper
inperf
impers
ind
indef
inf
ing
ingl
ins
intr
irreg
is
it
lat
lett
lav
loc
long
macch
mat
med
mediev
min
morf
mus
n
neg
neol
neur
nom
oc
occ
od
ogg
ord
ott
p
pag
part
partic
pass
pl
pol
pr
pref
prep
pres
pret
priv
prov
qlco
qlcu
qualif
rar
recipr
reg
region
rel
rem
rep
retor
rifl
rom
s
scherz
sec
secc
seg
segg
sig
sigg
sig.ra
sig.na
simb
sin
sing
sogg
sp
st
stat
suff
sup
superl
tav
tecn
tecnol
tel
ted
temp
term
tosc
tr
trad
trasp
v
voc
vol
volg
voll




[EOSMARKERS]
# Character: !
# Name: EXCLAMATION MARK
# Code: 33 (0x21)
\u0021

# Character: ?
# Name: QUESTION MARK
# Code: 3f (0x3f)
\u003F

# Character: ;
# Name: GREEK QUESTION MARK
# Code: 894 (0x37e)
\u037e

# Character: ؟
# Name: ARABIC QUESTION MARK
# Code: 1567 (0x61f)
\u061f

# Character: 。
# Name: IDEOGRAPHIC FULL STOP
# Code: 12290 (0x3002)
\u3002

# Character: ｡
# Name: HALFWIDTH IDEOGRAPHIC FULL STOP
# Code: 65377 (0xff61)
\uff61

# Character: ？
# Name: FULLWIDTH QUESTION MARK
# Code: 65311 (0xff1f)
\uff1f

# Character: ！
# Name: FULLWIDTH EXCLAMATION MARK
# Code: 65281 (0xff01)
\uff01

# Character: ।
# Name: DEVANAGARI DANDA
# Code: 2404 (0x964)
\u0964

# Character: ։
# Name: ARMENIAN FULL STOP
# Code: 1417 (0x589)
\u0589

# Character: ՞
# Name: ARMENIAN QUESTION MARK
# Code: 1374 (0x55e)
\u055e

# Character: ።
# Name: ETHIOPIC FULL STOP
# Code: 4962 (0x1362)
\u1362

# Character: ᙮
# Name: CANADIAN SYLLABICS FULL STOP
# Code: 5742 (0x166e)
\u166e

# Character: ។
# Name: KHMER SIGN KHAN
# Code: 6100 (0x17d4)
\u17d4

# Character: ៕
# Name: KHMER SIGN BARIYOOSAN
# Code: 6101 (0x17d5)
\u17d5

# Character: ᠃
# Name: MONGOLIAN FULL STOP
# Code: 6147 (0x1803)
\u1803

# Character: ᠉
# Name: MONGOLIAN MANCHU FULL STOP
# Code: 6153 (0x1809)
\u1809
