1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
//=============================================================================
//
// File : libkvilanguage.cpp
// Creation date : Fri 25 Dec 20:16:49 2004 GMT by Szymon Stefanek
//
// This file is part of the KVirc irc client distribution
// Copyright (C) 2004 Szymon Stefanek (pragma at kvirc dot net)
//
// This program is FREE software. You can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your opinion) any later version.
//
// This program is distributed in the HOPE that it will be USEFUL,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, write to the Free Software Foundation,
// Inc. ,59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
//
//=============================================================================
#include "kvi_module.h"
#include "kvi_app.h"
#include "kvi_locale.h"
#include <QFileInfo>
#include "detector.h"
/*
@doc: language.detect
@type:
function
@title:
$language.detect
@short:
(Tries to) detect the language and encoding of a string
@syntax:
<hash> language.detect(<text: string>)
@description:
Tries to detect the language and encoding of the string specified
as [text]. The accuracy of the detection mainly depends on the length
of the supplied text. Good results can be achieved with some thousands
characters.[br]
This command is exported by the "language" module.
@examples:
[example]
%dati = $language.detect("I'm a lord and i speak perfect english.");
if(%dati{"error"} != "")echo "Language detection failed: " %dati{"error"};
%count = %dati{"matchcount"};
for(%i=0;%i<%count;%i++)
{
echo "LANGUAGE " %i " : " %dati{"matches"}[%i]{"language"};
echo "ENCODING " %i " : " %dati{"matches"}[%i]{"encoding"};
echo "SCORE " %i " : "%dati{"matches"}[%i]{"score"};
}
echo "ACCURACY: " %dati{"accuracy"};
[/example]
*/
static bool language_kvs_cmd_detect(KviKvsModuleFunctionCall * c)
{
QString text, error;
KviKvsArray *matches;
KviKvsHash *match, *ret;
LanguageAndEncodingResult r;
int matchcount=DLE_NUM_BEST_MATCHES;
KVSM_PARAMETERS_BEGIN(c)
KVSM_PARAMETER("text",KVS_PT_STRING,KVS_PF_OPTIONAL,text)
KVSM_PARAMETERS_END(c)
//on error we return an empty array
matches = new KviKvsArray();
if(text.isEmpty()){
error = "err_notext";
matchcount = 0;
} else {
detect_language_and_encoding(text.utf8().data(),&r,0);
for(int i=0;i<matchcount;i++)
{
match = new KviKvsHash();
match->set("language",new KviKvsVariant(QString(r.match[i].szLanguage)));
match->set("encoding",new KviKvsVariant(r.match[i].szEncoding));
match->set("score",new KviKvsVariant((kvs_real_t)r.match[i].dScore));
matches->set(i, new KviKvsVariant(match));
}
}
ret = new KviKvsHash();
ret->set("matches",new KviKvsVariant(matches));
ret->set("matchcount",new KviKvsVariant((kvs_int_t)matchcount));
ret->set("error",new KviKvsVariant(error));
ret->set("accuracy",new KviKvsVariant((kvs_real_t)r.dAccuracy));
c->returnValue()->setHash(ret);
return true;
}
static bool language_module_init(KviModule * m)
{
KVSM_REGISTER_FUNCTION(m,"detect",language_kvs_cmd_detect);
return true;
}
static bool language_module_cleanup(KviModule *m)
{
return true;
}
KVIRC_MODULE(
"Language", // module name
"4.0.0", // module version
"Copyright (C) 2004 Szymon Stefanek (pragma at kvirc dot net)", // author & (C)
"A simple statistical language/encoding detector",
language_module_init,
0,
0,
language_module_cleanup
)
|
