1 package org.daisy.pipeline.nlp.ruledcategorizers;
2
3 import java.io.IOException;
4
5 import org.daisy.pipeline.nlp.RuleBasedTextCategorizer;
6 import org.daisy.pipeline.nlp.impl.matchrules.NumberRangeMatchRule;
7 import org.daisy.pipeline.nlp.impl.matchrules.RegexMatchRule;
8
9 public class RuledMultilangCategorizer extends RuleBasedTextCategorizer {
10
11 public static int LOWEST_PRIORITY = 0;
12 public static int COMMON_WORD_MAX_PRIORITY = 50;
13 public static int SPACE_MAX_PRIORITY = 100;
14 public static int QUOTE_MAX_PRIORITY = 125;
15 public static int NUMBER_MAX_PRIORITY = 150;
16 public static int ACRONYM_MAX_PRIORITY = 160;
17 public static int ABBR_MAX_PRIORITY = 220;
18 public static int WEBLINK_MAX_PRIORITY = 300;
19 public static int SPACE_COMPOSED_MAX_PRIORITY = 500;
20 public static int NUMBER_COMPOSED_MAX_PRIORITY = 600;
21 public static int DICTIONARY_MAX_PRIORITY = 700;
22 protected static String CommonWordPattern = "[@\\p{L}][-_@\\p{L}\\p{Nd}]*";
23
24
25 protected static String Space = "";
26 static {
27 char[] SpaceChars = {
28 0x0020, 0x0085, 0x00A0, 0x1680, 0x180E, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000
29 };
30 for (char spaceChar : SpaceChars) {
31 Space += new Character(spaceChar);
32 }
33 Space += new Character((char) 0x0009) + "-" + new Character((char) 0x000D);
34 Space += new Character((char) 0x2000) + "-" + new Character((char) 0x200A);
35 }
36
37 @Override
38 public void init(MatchMode matchMode) throws IOException {
39 super.init(matchMode);
40
41 RegexMatchRule rsm;
42
43
44 String year = "([1-9][0-9]{1,3}|[0-9]{2})";
45 String month = "(1[0-2]|0?[1-9])";
46 String days = "(3[01]|[12]0|[0-2]?[1-9])";
47
48 rsm = new RegexMatchRule(Category.DATE, NUMBER_COMPOSED_MAX_PRIORITY, true, mMatchMode);
49 rsm.init(year + "-" + month + "-" + days + "(?![-\\p{L}\\p{Nd}])");
50 addRule(rsm);
51
52 rsm = new RegexMatchRule(Category.DATE, NUMBER_COMPOSED_MAX_PRIORITY, true, mMatchMode);
53 rsm.init(month + "-" + days + "(?![-\\p{L}\\p{Nd}])");
54 addRule(rsm);
55
56 rsm = new RegexMatchRule(Category.DATE, NUMBER_COMPOSED_MAX_PRIORITY, true, mMatchMode);
57 rsm.init(days + "/" + month + "/" + year + "(?![/\\p{L}\\p{Nd}])");
58 addRule(rsm);
59
60 rsm = new RegexMatchRule(Category.DATE, NUMBER_COMPOSED_MAX_PRIORITY, true, mMatchMode);
61 rsm.init(days + "/" + month + "(?![/\\p{L}\\p{Nd}])");
62 addRule(rsm);
63
64 addRule(new NumberRangeMatchRule(Category.RANGE, NUMBER_COMPOSED_MAX_PRIORITY,
65 mMatchMode));
66
67 rsm = new RegexMatchRule(Category.DATE, NUMBER_COMPOSED_MAX_PRIORITY, true, mMatchMode);
68 rsm.init(year + "-" + month + "(?![-\\p{L}\\p{Nd}])");
69 addRule(rsm);
70
71 rsm = new RegexMatchRule(Category.DATE, NUMBER_COMPOSED_MAX_PRIORITY, true, mMatchMode);
72 rsm.init(month + "/" + year + "(?![/\\p{L}\\p{Nd}])");
73 addRule(rsm);
74
75
76 rsm = new RegexMatchRule(Category.TIME, NUMBER_COMPOSED_MAX_PRIORITY, true, mMatchMode);
77 rsm.init("(2[0-4]|[01][0-9]):[0-6][0-9](?![0-9])");
78 addRule(rsm);
79
80
81
82
83 String integer = "([1-9]{1,3}([,' ][0-9]{3})+|[1-9][0-9]*)";
84 String real = "(" + integer + "(\\.[0-9]+)?)";
85 String currency = "([\\$€£₤¥]|usd|euro[s]?)";
86
87 rsm = new RegexMatchRule(Category.DIMENSIONS, NUMBER_COMPOSED_MAX_PRIORITY, true,
88 mMatchMode);
89 rsm.init(real + "(x|[ ]x[ ])" + real + "(?![\\p{L}\\p{Nd}])");
90 addRule(rsm);
91
92 rsm = new RegexMatchRule(Category.CURRENCY, NUMBER_COMPOSED_MAX_PRIORITY, false,
93 mMatchMode);
94 rsm.init(real + currency + "(?![\\p{L}\\p{Nd}])");
95 addRule(rsm);
96
97 rsm = new RegexMatchRule(Category.CURRENCY, NUMBER_COMPOSED_MAX_PRIORITY, false,
98 mMatchMode);
99 rsm.init(currency + real + "(?![\\p{L}\\p{Nd}])");
100 addRule(rsm);
101
102 rsm = new RegexMatchRule(Category.NUMBERING_ITEM, NUMBER_COMPOSED_MAX_PRIORITY, false,
103 mMatchMode);
104 rsm.init("[0-9]+([-.][0-9]+)*\\." + "(?![\\p{L}\\p{Nd}])");
105 addRule(rsm);
106
107 rsm = new RegexMatchRule(Category.QUANTITY, NUMBER_MAX_PRIORITY, true, mMatchMode);
108 rsm.init(real);
109 addRule(rsm);
110
111 rsm = new RegexMatchRule(Category.IDENTIFIER, NUMBER_MAX_PRIORITY - 1, true,
112 mMatchMode);
113 rsm.init("[0-9]+([-_:][0-9]+)*" + "(?![\\p{L}\\p{Nd}])");
114 addRule(rsm);
115
116
117 rsm = new RegexMatchRule(Category.SPACE, SPACE_MAX_PRIORITY, true, mMatchMode);
118 rsm.init("[" + Space + "]+");
119 addRule(rsm);
120
121
122 rsm = new RegexMatchRule(Category.QUOTE, QUOTE_MAX_PRIORITY, true, mMatchMode);
123 rsm.init("[\\p{Pf}\\p{Pi}\"']");
124 addRule(rsm);
125
126
127 rsm = new RegexMatchRule(Category.WEB_LINK, WEBLINK_MAX_PRIORITY, true, mMatchMode);
128 rsm.init("[a-z]+://[^" + Space + "]*");
129 addRule(rsm);
130
131 rsm = new RegexMatchRule(Category.WEB_LINK, WEBLINK_MAX_PRIORITY, true, mMatchMode);
132 rsm.init("www\\.[^" + Space + "]+");
133 addRule(rsm);
134
135 rsm = new RegexMatchRule(Category.EMAIL_ADDR, WEBLINK_MAX_PRIORITY, true, mMatchMode);
136 rsm.init("[\\p{L}][-_.\\p{L}\\p{Nd}]*(@|\\(at\\))[\\p{L}][-_.\\p{L}\\p{Nd}]*");
137 addRule(rsm);
138
139
140
141 String acronymPrefix = "[\\p{L}]\\.([-]?[\\p{L}\\p{Nd}]\\.)+";
142
143
144
145
146
147 rsm = new RegexMatchRule(Category.ACRONYM, ACRONYM_MAX_PRIORITY, true, mMatchMode);
148 rsm.init(acronymPrefix + "(?=[" + Space + "]+[\\p{Ll}])");
149 addRule(rsm);
150
151
152 rsm = new RegexMatchRule(Category.ACRONYM, ACRONYM_MAX_PRIORITY, true, mMatchMode);
153 rsm.init(acronymPrefix + "[\\p{L}\\p{Nd}]");
154 addRule(rsm);
155
156
157
158
159 rsm = new RegexMatchRule(Category.ACRONYM, ACRONYM_MAX_PRIORITY, true, mMatchMode);
160 rsm.init("[\\p{Lu}]\\.(?=[" + Space + "])");
161 addRule(rsm);
162
163
164 rsm = new RegexMatchRule(Category.ABBREVIATION, ABBR_MAX_PRIORITY, true, mMatchMode);
165 rsm.init("[\\p{Ll}]\\.(?=[" + Space + "])");
166 addRule(rsm);
167
168
169 rsm = new RegexMatchRule(Category.COMMON, COMMON_WORD_MAX_PRIORITY, true, mMatchMode);
170 rsm.init(CommonWordPattern);
171 addRule(rsm);
172
173
174 rsm = new RegexMatchRule(Category.PUNCTUATION, LOWEST_PRIORITY, true, mMatchMode);
175 rsm.init(".");
176 addRule(rsm);
177 }
178 }