aboutsummaryrefslogtreecommitdiffstats
path: root/src/modules
diff options
context:
space:
mode:
Diffstat (limited to 'src/modules')
-rw-r--r--src/modules/CMakeLists.txt4
-rw-r--r--src/modules/language/CMakeLists.txt18
-rw-r--r--src/modules/language/detector.cpp39
-rw-r--r--src/modules/language/detector.h43
-rwxr-xr-xsrc/modules/language/detector/build.sh5
-rwxr-xr-xsrc/modules/language/detector/build_detector.pl49
-rw-r--r--src/modules/language/langdetector.cpp46
-rw-r--r--src/modules/language/langdetector.h37
-rw-r--r--src/modules/language/libkvilanguage.cpp63
9 files changed, 236 insertions, 68 deletions
diff --git a/src/modules/CMakeLists.txt b/src/modules/CMakeLists.txt
index b5c433c92..1d6712de3 100644
--- a/src/modules/CMakeLists.txt
+++ b/src/modules/CMakeLists.txt
@@ -27,7 +27,7 @@ SUBDIRS(
file filetransferwindow
help http
ident iograph
- lamerizer links list log logview
+ lamerizer language links list log logview
mask math mediaplayer mircimport my
notifier
objects options
@@ -46,5 +46,5 @@ SUBDIRS(
# perl, perlcore
# forgotten ?
- # language offer
+ # offer
)
diff --git a/src/modules/language/CMakeLists.txt b/src/modules/language/CMakeLists.txt
new file mode 100644
index 000000000..501d16c78
--- /dev/null
+++ b/src/modules/language/CMakeLists.txt
@@ -0,0 +1,18 @@
+# CMakeLists for src/modules/language
+
+SET(kvilanguage_MOC_HDRS
+ detector.h
+ langdetector.h
+)
+
+SET(kvilanguage_SRCS
+ libkvilanguage.cpp
+ detector.cpp
+ langdetector.cpp
+)
+
+# After this call, files will be moc'ed to moc_kvi_*.cpp
+QT4_WRAP_CPP(kvilanguage_MOC_SRCS ${kvilanguage_MOC_HDRS})
+
+SET(kvi_module_name kvilanguage)
+INCLUDE(${CMAKE_SOURCE_DIR}/cmake/module.rules.txt)
diff --git a/src/modules/language/detector.cpp b/src/modules/language/detector.cpp
index bc5391fca..f396f6d0e 100644
--- a/src/modules/language/detector.cpp
+++ b/src/modules/language/detector.cpp
@@ -28,6 +28,8 @@
#include <string.h>
#include <ctype.h>
+#include "detector.h"
+
///////////////////////////////////////////////////////////////////////////////
// CORE DEFS
///////////////////////////////////////////////////////////////////////////////
@@ -7396,7 +7398,7 @@ D l13_d =
//
// Lng: finnish
-// Enc: iso-8859-1
+// Enc: iso-8859-15
//
@@ -7779,7 +7781,7 @@ N l14n255[]={{sqyy,.109},H};
D l14_d =
{
"finnish",
- "iso-8859-1",
+ "iso-8859-15",
{
W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
@@ -7822,7 +7824,7 @@ D l14_d =
//
// Lng: french
-// Enc: iso-8859-1
+// Enc: iso-8859-15
//
@@ -8137,7 +8139,7 @@ N l15n255[]={{szi,.235},{sXtt,.149},{ssvx,.130},H};
D l15_d =
{
"french",
- "iso-8859-1",
+ "iso-8859-15",
{
W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
@@ -17780,7 +17782,7 @@ D l36_d =
//
// Lng: spanish
-// Enc: iso-8859-1
+// Enc: iso-8859-15
//
@@ -18048,7 +18050,7 @@ N l37n255[]={{szi,.184},{smsri,.221},H};
D l37_d =
{
"spanish",
- "iso-8859-1",
+ "iso-8859-15",
{
W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
@@ -18091,7 +18093,7 @@ D l37_d =
//
// Lng: swedish
-// Enc: iso-8859-1
+// Enc: iso-8859-15
//
@@ -18370,7 +18372,7 @@ N l38n255[]={{szi,.173},{smsri,.333},{sg2f,.182},H};
D l38_d =
{
"swedish",
- "iso-8859-1",
+ "iso-8859-15",
{
W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
@@ -19048,23 +19050,6 @@ static int utf8score(const unsigned char * p)
}
-typedef struct _LanguageAndEncodingMatch
-{
- const char * szLanguage;
- const char * szEncoding;
- double dScore;
-} LanguageAndEncodingMatch;
-
-#define DLE_NUM_BEST_MATCHES 4
-
-typedef struct _LanguageAndEncodingResult
-{
- LanguageAndEncodingMatch match[DLE_NUM_BEST_MATCHES]; // the first best matches
- double dAccuracy; // accuracy score: from 0 to 100
-} LanguageAndEncodingResult;
-
-#define DLE_STRICT_UTF8_CHECKING 1
-
static const char * unknown_string = "?";
void detect_language_and_encoding(const char * data,LanguageAndEncodingResult * retBuffer,int iFlags = 0)
@@ -19130,6 +19115,8 @@ void detect_language_and_encoding(const char * data,LanguageAndEncodingResult *
else retBuffer->dAccuracy = 0.0;
}
+/*
+ * this file can be compiled also as a standalone app for testing
int main(int argc,char ** argv)
{
FILE * f = fopen(argv[1],"r");
@@ -19145,3 +19132,5 @@ int main(int argc,char ** argv)
printf("Accuracy: %f\n",r.dAccuracy);
return 0;
}
+*/
+
diff --git a/src/modules/language/detector.h b/src/modules/language/detector.h
new file mode 100644
index 000000000..6131af8ff
--- /dev/null
+++ b/src/modules/language/detector.h
@@ -0,0 +1,43 @@
+#ifndef _DETECTOR_H_
+#define _DETECTOR_H_
+//
+// File : detector.h
+// Creation date : Sun May 11 2008 00:07:20 CET by Szymon Stefanek
+//
+// This file is part of the KVirc irc client distribution
+// Copyright (C) 1999-2008 Szymon Stefanek (pragma at kvirc dot net)
+//
+// This program is FREE software. You can redistribute it and/or
+// modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 2
+// of the License, or (at your opinion) any later version.
+//
+// This program is distributed in the HOPE that it will be USEFUL,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+// See the GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, write to the Free Software Foundation,
+// Inc. ,59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+//
+
+#define DLE_NUM_BEST_MATCHES 4
+#define DLE_STRICT_UTF8_CHECKING 1
+
+typedef struct _LanguageAndEncodingMatch
+{
+ const char * szLanguage;
+ const char * szEncoding;
+ double dScore;
+} LanguageAndEncodingMatch;
+
+typedef struct _LanguageAndEncodingResult
+{
+ LanguageAndEncodingMatch match[DLE_NUM_BEST_MATCHES]; // the first best matches
+ double dAccuracy; // accuracy score: from 0 to 100
+} LanguageAndEncodingResult;
+
+void detect_language_and_encoding(const char * data,LanguageAndEncodingResult * retBuffer,int iFlags);
+
+#endif //_DETECTOR_H_
diff --git a/src/modules/language/detector/build.sh b/src/modules/language/detector/build.sh
index 47e36f1a1..121750cdb 100755
--- a/src/modules/language/detector/build.sh
+++ b/src/modules/language/detector/build.sh
@@ -1,4 +1,7 @@
#!/bin/bash
+#generate detecting routines and tables
perl build_detector.pl ../detector.cpp
-gcc -o detector ../detector.cpp
+
+#compile detector.cpp as a standalone app (only for testing)
+#g++ -o detector ../detector.cpp
diff --git a/src/modules/language/detector/build_detector.pl b/src/modules/language/detector/build_detector.pl
index 96aaf98a8..9f2a073ae 100755
--- a/src/modules/language/detector/build_detector.pl
+++ b/src/modules/language/detector/build_detector.pl
@@ -26,7 +26,7 @@
#
#
# Original copyright notice:
-# © Gertjan van Noord, 1997.
+# © Gertjan van Noord, 1997.
# mailto:vannoord@let.rug.nl
#
#
@@ -99,6 +99,8 @@ print OUTPUT "#include <stdio.h>\n";
print OUTPUT "#include <string.h>\n";
print OUTPUT "#include <ctype.h>\n";
print OUTPUT "\n";
+print OUTPUT "#include \"detector.h\"\n";
+print OUTPUT "\n";
print OUTPUT "///////////////////////////////////////////////////////////////////////////////\n";
print OUTPUT "// CORE DEFS\n";
print OUTPUT "///////////////////////////////////////////////////////////////////////////////\n";
@@ -283,7 +285,7 @@ foreach $file (@g_files)
# since they are characteristic of language (and we thus probably WANT
# to include them in the hash
# Chars above 191 get yet more
-
+
foreach(@result)
{
if((ord $_) > 127)
@@ -296,7 +298,7 @@ foreach $file (@g_files)
}
}
- splice(@result,$g_maxNGrams) if (@result > $g_maxNGrams);
+ splice(@result,$g_maxNGrams) if (@result > $g_maxNGrams);
my $results = 0;
my $ngrams = 0;
@@ -336,7 +338,7 @@ foreach $file (@g_files)
my @strings = ();
my @string_names = ();
my $string_count = 0;
-
+
for($i=0;$i<256;$i++)
{
$lists[$i] = "";
@@ -353,7 +355,7 @@ foreach $file (@g_files)
my($hash,$j,$len);
$len = length($_);
$lenstats[$len]++;
-
+
$hash = (ord substr($_,0,1)) * 31;
if($len > 1)
{
@@ -368,7 +370,7 @@ foreach $file (@g_files)
{
$hash = $hash + ord substr($_,4,1);
}
- }
+ }
}
}
@@ -381,7 +383,7 @@ foreach $file (@g_files)
my $string_name = "s";
for($z = 0;$z < length($_);$z++)
{
-
+
if($tmp ne ""){ $tmp .= ","; };
my $zzz1 = substr($_,$z,1);
my $zzz2 = ord $zzz1;
@@ -395,7 +397,7 @@ foreach $file (@g_files)
}
$tmp .= ",0";
-
+
$tmp =~ s/,32,0/,a/g; # really common sequence
$strings[$string_count] = $tmp;
@@ -422,7 +424,7 @@ foreach $file (@g_files)
print OUTPUT "S ".$string_names[$i]."[]={".$strings[$i]."};\n";
}
}
-
+
print OUTPUT "\n";
# now print the lists
@@ -463,12 +465,12 @@ foreach $file (@g_files)
} else {
$perc = 0.0;
}
-
+
if($debug > 0){ print OUTPUT " "; }
else {
if(($i % 16) == 0){ print OUTPUT " "; }
}
-
+
if($perc == 0.0){ print OUTPUT "Z"; }
elsif($perc == -0.100){ print OUTPUT "W"; }
elsif($perc == -0.250){ print OUTPUT "Y"; }
@@ -489,7 +491,7 @@ foreach $file (@g_files)
for($i=0;$i<256;$i++)
{
if(($i % 16) == 0){ print OUTPUT " "; };
-
+
if($lists[$i] ne "")
{
print OUTPUT "".$m_tableNamePrefix."n".$i;
@@ -782,23 +784,6 @@ print OUTPUT "\n";
print OUTPUT "\n";
-print OUTPUT "typedef struct _LanguageAndEncodingMatch\n";
-print OUTPUT "{\n";
-print OUTPUT " const char * szLanguage;\n";
-print OUTPUT " const char * szEncoding;\n";
-print OUTPUT " double dScore;\n";
-print OUTPUT "} LanguageAndEncodingMatch;\n";
-print OUTPUT "\n";
-print OUTPUT "#define DLE_NUM_BEST_MATCHES 4\n";
-print OUTPUT "\n";
-print OUTPUT "typedef struct _LanguageAndEncodingResult\n";
-print OUTPUT "{\n";
-print OUTPUT " LanguageAndEncodingMatch match[DLE_NUM_BEST_MATCHES]; // the first best matches\n";
-print OUTPUT " double dAccuracy; // accuracy score: from 0 to 100\n";
-print OUTPUT "} LanguageAndEncodingResult;\n";
-print OUTPUT "\n";
-print OUTPUT "#define DLE_STRICT_UTF8_CHECKING 1\n";
-print OUTPUT "\n";
print OUTPUT "static const char * unknown_string = \"?\";\n";
print OUTPUT "\n";
print OUTPUT "void detect_language_and_encoding(const char * data,LanguageAndEncodingResult * retBuffer,int iFlags = 0)\n";
@@ -878,7 +863,8 @@ print OUTPUT " else retBuffer->dAccuracy = 0.0;\n";
print OUTPUT "}\n";
print OUTPUT "\n";
-
+print OUTPUT "/*\n";
+print OUTPUT " * this file can be compiled also as a standalone app for testing\n";
print OUTPUT "int main(int argc,char ** argv)\n";
print OUTPUT "{\n";
print OUTPUT " FILE * f = fopen(argv[1],\"r\");\n";
@@ -894,7 +880,8 @@ print OUTPUT " printf(\"LANGUAGE %s, ENCODING %s, SCORE: %f\\n\",r.match[i].szL
print OUTPUT " printf(\"Accuracy: %f\\n\",r.dAccuracy);\n";
print OUTPUT " return 0;\n";
print OUTPUT "}\n";
-
+print OUTPUT "*/\n";
+print OUTPUT "\n";
close(OUTPUT);
diff --git a/src/modules/language/langdetector.cpp b/src/modules/language/langdetector.cpp
new file mode 100644
index 000000000..979e840f1
--- /dev/null
+++ b/src/modules/language/langdetector.cpp
@@ -0,0 +1,46 @@
+//
+// File : helpwindow.cpp
+// Creation date : Tue Aug 11 2000 18:08:22 by Szymon Stefanek
+//
+// This file is part of the KVirc irc client distribution
+// Copyright (C) 1999-2000 Szymon Stefanek (pragma at kvirc dot net)
+//
+// This program is FREE software. You can redistribute it and/or
+// modify it under the m_terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 2
+// of the License, or (at your opinion) any later version.
+//
+// This program is distributed in the HOPE that it will be USEFUL,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+// See the GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, write to the Free Software Foundation,
+// Inc. ,59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+//
+
+#include <QFile>
+
+#include "langdetector.h"
+
+KviLangDetetector::KviLangDetetector(QString doc)
+{
+ qint64 iMaxSize = 4096;
+ char buffer[4096];
+ QFile *file = new QFile(doc);
+
+ if (!file->open(QIODevice::ReadOnly | QIODevice::Text))
+ return;
+
+ if(file->read(buffer, iMaxSize)<1)
+ return;
+
+ LanguageAndEncodingResult r;
+ detect_language_and_encoding(buffer,&r,0);
+ for(int i=0;i<DLE_NUM_BEST_MATCHES;i++)
+ printf("LANGUAGE %s, ENCODING %s, SCORE: %f\n",r.match[i].szLanguage,r.match[i].szEncoding,r.match[i].dScore);
+ printf("Accuracy: %f\n",r.dAccuracy);
+ return;
+}
+
diff --git a/src/modules/language/langdetector.h b/src/modules/language/langdetector.h
new file mode 100644
index 000000000..e9beeae28
--- /dev/null
+++ b/src/modules/language/langdetector.h
@@ -0,0 +1,37 @@
+#ifndef _LANGDETECTOR_H_
+#define _LANGDETECTOR_H_
+//
+// File : langdetector.h
+// Creation date : Sun May 11 2008 00:07:20 CET by Szymon Stefanek
+//
+// This file is part of the KVirc irc client distribution
+// Copyright (C) 1999-2008 Szymon Stefanek (pragma at kvirc dot net)
+//
+// This program is FREE software. You can redistribute it and/or
+// modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 2
+// of the License, or (at your opinion) any later version.
+//
+// This program is distributed in the HOPE that it will be USEFUL,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+// See the GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, write to the Free Software Foundation,
+// Inc. ,59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+//
+
+#include "kvi_string.h"
+#include "kvi_kvs_array.h"
+
+#include "detector.h"
+
+class KviLangDetetector
+{
+public:
+ KviLangDetetector(QString doc);
+ ~KviLangDetetector();
+};
+
+#endif //_LANGDETECTOR_H_
diff --git a/src/modules/language/libkvilanguage.cpp b/src/modules/language/libkvilanguage.cpp
index c7ce80a9c..c2fe14a64 100644
--- a/src/modules/language/libkvilanguage.cpp
+++ b/src/modules/language/libkvilanguage.cpp
@@ -24,19 +24,64 @@
#include "kvi_module.h"
#include "kvi_app.h"
+#include "kvi_locale.h"
+#include <QFileInfo>
+
+#include "langdetector.h"
+
+/*
+ @doc: language.detect
+ @type:
+ command
+ @title:
+ language.detect
+ @short:
+ (Tries to) detect the language and encoding of a text file
+ @syntax:
+ language.detect [document: string]
+ @description:
+ Tries to detect the language and encoding of the file specified
+ as [document]. The file has to be preferably a simple text file,
+ or the detector can be deceived by the file format (eg. html files
+ contains english words inside html tags).
+ [document] can be an absolute path.
+ This command is exported by the "language" module.
+*/
+
+
+static bool language_kvs_cmd_detect(KviKvsModuleCommandCall * c)
+{
+ QString doc;
+
+ KVSM_PARAMETERS_BEGIN(c)
+ KVSM_PARAMETER("document",KVS_PT_STRING,KVS_PF_OPTIONAL,doc)
+ KVSM_PARAMETERS_END(c)
+
+ if(doc.isEmpty()){
+ c->warning(__tr2qs("No file given, no detection occured."));
+ return true;
+ }
+
+ QFileInfo * f= new QFileInfo(doc);
+ qDebug("Path %s",doc.toUtf8().data());
+ if(f)
+ {
+ if(!f->exists())
+ {
+ c->warning(__tr2qs("The file you specified doesn't exists, can't detect its language."));
+ return true;
+ }
+ }
+
+ KviLangDetetector *w = new KviLangDetetector(doc);
+
+ return true;
+}
static bool language_module_init(KviModule * m)
{
- /*
- m->registerFunction("nick",my_module_fnc_nick);
- m->registerFunction("user",my_module_fnc_user);
- m->registerFunction("host",my_module_fnc_host);
- m->registerFunction("ip",my_module_fnc_ip);
- m->registerFunction("server",my_module_fnc_server);
- m->registerFunction("network",my_module_fnc_network);
- m->registerFunction("umode",my_module_fnc_umode);
- */
+ KVSM_REGISTER_SIMPLE_COMMAND(m,"detect",language_kvs_cmd_detect);
return true;
}
generated by cgit v1.3.1-10-gc9f91 (git 2.54.0) at 2026-08-08 07:44:46 +0000