re-activated language module and started implementing its functions

git-svn-id: https://svn.kvirc.de/svn/trunk/kvirc@1743 17fca916-40b9-46aa-a4ea-0a15b648b75c

10 files changed, 237 insertions, 69 deletions

diff --git a/TODO-Qt4 b/TODO-Qt4
index da4d27cd7..62cf0b263 100644
--- a/TODO-Qt4
+++ b/TODO-Qt4
@@ -48,7 +48,7 @@ src/modules
 	clock - all excluded by #if
 	editor - q3support
 	iograph - all excluded by #if
-	language - doesn't export anything
+	language - has to be finished
 	notifier
 	objects - q3support
 	options - optw_protection.cpp bugged
diff --git a/src/modules/CMakeLists.txt b/src/modules/CMakeLists.txt
index b5c433c92..1d6712de3 100644
--- a/src/modules/CMakeLists.txt
+++ b/src/modules/CMakeLists.txt
@@ -27,7 +27,7 @@ SUBDIRS(
 	file filetransferwindow
 	help http
 	ident iograph
-	lamerizer links list log logview
+	lamerizer language links list log logview
 	mask math mediaplayer mircimport my
 	notifier
 	objects options
@@ -46,5 +46,5 @@ SUBDIRS(
 	# perl, perlcore
 
 	# forgotten ?
-	# language offer
+	# offer
 )
diff --git a/src/modules/language/CMakeLists.txt b/src/modules/language/CMakeLists.txt
new file mode 100644
index 000000000..501d16c78
--- /dev/null
+++ b/src/modules/language/CMakeLists.txt
@@ -0,0 +1,18 @@
+# CMakeLists for src/modules/language
+
+SET(kvilanguage_MOC_HDRS
+	detector.h
+	langdetector.h
+)
+
+SET(kvilanguage_SRCS
+	libkvilanguage.cpp
+	detector.cpp
+	langdetector.cpp
+)
+
+# After this call, files will be moc'ed to moc_kvi_*.cpp
+QT4_WRAP_CPP(kvilanguage_MOC_SRCS ${kvilanguage_MOC_HDRS})
+
+SET(kvi_module_name kvilanguage)
+INCLUDE(${CMAKE_SOURCE_DIR}/cmake/module.rules.txt)
diff --git a/src/modules/language/detector.cpp b/src/modules/language/detector.cpp
index bc5391fca..f396f6d0e 100644
--- a/src/modules/language/detector.cpp
+++ b/src/modules/language/detector.cpp
@@ -28,6 +28,8 @@
 #include <string.h>
 #include <ctype.h>
 
+#include "detector.h"
+
 ///////////////////////////////////////////////////////////////////////////////
 // CORE DEFS
 ///////////////////////////////////////////////////////////////////////////////
@@ -7396,7 +7398,7 @@ D l13_d =
 
 //
 // Lng: finnish
-// Enc: iso-8859-1
+// Enc: iso-8859-15
 //
 
 
@@ -7779,7 +7781,7 @@ N l14n255[]={{sqyy,.109},H};
 D l14_d =
 {
 	"finnish",
-	"iso-8859-1",
+	"iso-8859-15",
 	{
 		W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
 		W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
@@ -7822,7 +7824,7 @@ D l14_d =
 
 //
 // Lng: french
-// Enc: iso-8859-1
+// Enc: iso-8859-15
 //
 
 
@@ -8137,7 +8139,7 @@ N l15n255[]={{szi,.235},{sXtt,.149},{ssvx,.130},H};
 D l15_d =
 {
 	"french",
-	"iso-8859-1",
+	"iso-8859-15",
 	{
 		W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
 		W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
@@ -17780,7 +17782,7 @@ D l36_d =
 
 //
 // Lng: spanish
-// Enc: iso-8859-1
+// Enc: iso-8859-15
 //
 
 
@@ -18048,7 +18050,7 @@ N l37n255[]={{szi,.184},{smsri,.221},H};
 D l37_d =
 {
 	"spanish",
-	"iso-8859-1",
+	"iso-8859-15",
 	{
 		W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
 		W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
@@ -18091,7 +18093,7 @@ D l37_d =
 
 //
 // Lng: swedish
-// Enc: iso-8859-1
+// Enc: iso-8859-15
 //
 
 
@@ -18370,7 +18372,7 @@ N l38n255[]={{szi,.173},{smsri,.333},{sg2f,.182},H};
 D l38_d =
 {
 	"swedish",
-	"iso-8859-1",
+	"iso-8859-15",
 	{
 		W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
 		W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,W,
@@ -19048,23 +19050,6 @@ static int utf8score(const unsigned char * p)
 }
 
 
-typedef struct _LanguageAndEncodingMatch
-{
-	const char * szLanguage;
-	const char * szEncoding;
-	double dScore;
-} LanguageAndEncodingMatch;
-
-#define DLE_NUM_BEST_MATCHES 4
-
-typedef struct _LanguageAndEncodingResult
-{
-	LanguageAndEncodingMatch match[DLE_NUM_BEST_MATCHES]; // the first best matches
-	double dAccuracy; // accuracy score: from 0 to 100
-} LanguageAndEncodingResult;
-
-#define DLE_STRICT_UTF8_CHECKING 1
-
 static const char * unknown_string = "?";
 
 void detect_language_and_encoding(const char * data,LanguageAndEncodingResult * retBuffer,int iFlags = 0)
@@ -19130,6 +19115,8 @@ void detect_language_and_encoding(const char * data,LanguageAndEncodingResult *
 	else retBuffer->dAccuracy = 0.0;
 }
 
+/*
+ * this file can be compiled also as a standalone app for testing
 int main(int argc,char ** argv)
 {
 	FILE * f = fopen(argv[1],"r");
@@ -19145,3 +19132,5 @@ int main(int argc,char ** argv)
 	printf("Accuracy: %f\n",r.dAccuracy);
 	return 0;
 }
+*/
+
diff --git a/src/modules/language/detector.h b/src/modules/language/detector.h
new file mode 100644
index 000000000..6131af8ff
--- /dev/null
+++ b/src/modules/language/detector.h
@@ -0,0 +1,43 @@
+#ifndef _DETECTOR_H_
+#define _DETECTOR_H_
+//
+//   File : detector.h
+//   Creation date : Sun May 11 2008 00:07:20 CET by Szymon Stefanek
+//
+//   This file is part of the KVirc irc client distribution
+//   Copyright (C) 1999-2008 Szymon Stefanek (pragma at kvirc dot net)
+//
+//   This program is FREE software. You can redistribute it and/or
+//   modify it under the terms of the GNU General Public License
+//   as published by the Free Software Foundation; either version 2
+//   of the License, or (at your opinion) any later version.
+//
+//   This program is distributed in the HOPE that it will be USEFUL,
+//   but WITHOUT ANY WARRANTY; without even the implied warranty of
+//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//   See the GNU General Public License for more details.
+//
+//   You should have received a copy of the GNU General Public License
+//   along with this program. If not, write to the Free Software Foundation,
+//   Inc. ,59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+//
+
+#define DLE_NUM_BEST_MATCHES 4
+#define DLE_STRICT_UTF8_CHECKING 1
+
+typedef struct _LanguageAndEncodingMatch
+{
+	const char * szLanguage;
+	const char * szEncoding;
+	double dScore;
+} LanguageAndEncodingMatch;
+
+typedef struct _LanguageAndEncodingResult
+{
+	LanguageAndEncodingMatch match[DLE_NUM_BEST_MATCHES]; // the first best matches
+	double dAccuracy; // accuracy score: from 0 to 100
+} LanguageAndEncodingResult;
+
+void detect_language_and_encoding(const char * data,LanguageAndEncodingResult * retBuffer,int iFlags);
+
+#endif //_DETECTOR_H_
diff --git a/src/modules/language/detector/build.sh b/src/modules/language/detector/build.sh
index 47e36f1a1..121750cdb 100755
--- a/src/modules/language/detector/build.sh
+++ b/src/modules/language/detector/build.sh
@@ -1,4 +1,7 @@
 #!/bin/bash
 
+#generate detecting routines and tables
 perl build_detector.pl ../detector.cpp
-gcc -o detector ../detector.cpp
+
+#compile detector.cpp as a standalone app (only for testing)
+#g++ -o detector ../detector.cpp
diff --git a/src/modules/language/detector/build_detector.pl b/src/modules/language/detector/build_detector.pl
index 96aaf98a8..9f2a073ae 100755
--- a/src/modules/language/detector/build_detector.pl
+++ b/src/modules/language/detector/build_detector.pl
@@ -26,7 +26,7 @@
 #
 #
 # Original copyright notice:
-# © Gertjan van Noord, 1997.
+# © Gertjan van Noord, 1997.
 # mailto:vannoord@let.rug.nl
 #
 #
@@ -99,6 +99,8 @@ print OUTPUT "#include <stdio.h>\n";
 print OUTPUT "#include <string.h>\n";
 print OUTPUT "#include <ctype.h>\n";
 print OUTPUT "\n";
+print OUTPUT "#include \"detector.h\"\n";
+print OUTPUT "\n";
 print OUTPUT "///////////////////////////////////////////////////////////////////////////////\n";
 print OUTPUT "// CORE DEFS\n";
 print OUTPUT "///////////////////////////////////////////////////////////////////////////////\n";
@@ -283,7 +285,7 @@ foreach $file (@g_files)
 	# since they are characteristic of language (and we thus probably WANT
 	# to include them in the hash
 	# Chars above 191 get yet more
-	
+
 	foreach(@result)
 	{
 		if((ord $_) > 127)
@@ -296,7 +298,7 @@ foreach $file (@g_files)
 		}
 	}
 
-	splice(@result,$g_maxNGrams) if (@result > $g_maxNGrams); 
+	splice(@result,$g_maxNGrams) if (@result > $g_maxNGrams);
 
 	my $results = 0;
 	my $ngrams = 0;
@@ -336,7 +338,7 @@ foreach $file (@g_files)
 	my @strings = ();
 	my @string_names = ();
 	my $string_count = 0;
-	
+
 	for($i=0;$i<256;$i++)
 	{
 		$lists[$i] = "";
@@ -353,7 +355,7 @@ foreach $file (@g_files)
 		my($hash,$j,$len);
 		$len = length($_);
 		$lenstats[$len]++;
-		
+
 		$hash = (ord substr($_,0,1)) * 31;
 		if($len > 1)
 		{
@@ -368,7 +370,7 @@ foreach $file (@g_files)
 					{
 						$hash = $hash + ord substr($_,4,1);
 					}
-				} 
+				}
 			}
 		}
 
@@ -381,7 +383,7 @@ foreach $file (@g_files)
 		my $string_name = "s";
 		for($z = 0;$z < length($_);$z++)
 		{
-			
+
 			if($tmp ne ""){ $tmp .= ","; };
 			my $zzz1 = substr($_,$z,1);
 			my $zzz2 = ord $zzz1;
@@ -395,7 +397,7 @@ foreach $file (@g_files)
 		}
 
 		$tmp .= ",0";
-		
+
 		$tmp =~ s/,32,0/,a/g; # really common sequence
 
 		$strings[$string_count] = $tmp;
@@ -422,7 +424,7 @@ foreach $file (@g_files)
 			print OUTPUT "S ".$string_names[$i]."[]={".$strings[$i]."};\n";
 		}
 	}
-	
+
 	print OUTPUT "\n";
 
 	# now print the lists
@@ -463,12 +465,12 @@ foreach $file (@g_files)
 		} else {
 			$perc = 0.0;
 		}
-		
+
 		if($debug > 0){ print OUTPUT "		"; }
 		else {
 			if(($i % 16) == 0){ print OUTPUT "		"; }
 		}
-		
+
 		if($perc == 0.0){ print OUTPUT "Z"; }
 		elsif($perc == -0.100){ print OUTPUT "W"; }
 		elsif($perc == -0.250){ print OUTPUT "Y"; }
@@ -489,7 +491,7 @@ foreach $file (@g_files)
 	for($i=0;$i<256;$i++)
 	{
 		if(($i % 16) == 0){ print OUTPUT "		"; };
-	
+
 		if($lists[$i] ne "")
 		{
 			print OUTPUT "".$m_tableNamePrefix."n".$i;
@@ -782,23 +784,6 @@ print OUTPUT "\n";
 
 
 print OUTPUT "\n";
-print OUTPUT "typedef struct _LanguageAndEncodingMatch\n";
-print OUTPUT "{\n";
-print OUTPUT "	const char * szLanguage;\n";
-print OUTPUT "	const char * szEncoding;\n";
-print OUTPUT "	double dScore;\n";
-print OUTPUT "} LanguageAndEncodingMatch;\n";
-print OUTPUT "\n";
-print OUTPUT "#define DLE_NUM_BEST_MATCHES 4\n";
-print OUTPUT "\n";
-print OUTPUT "typedef struct _LanguageAndEncodingResult\n";
-print OUTPUT "{\n";
-print OUTPUT "	LanguageAndEncodingMatch match[DLE_NUM_BEST_MATCHES]; // the first best matches\n";
-print OUTPUT "	double dAccuracy; // accuracy score: from 0 to 100\n";
-print OUTPUT "} LanguageAndEncodingResult;\n";
-print OUTPUT "\n";
-print OUTPUT "#define DLE_STRICT_UTF8_CHECKING 1\n";
-print OUTPUT "\n";
 print OUTPUT "static const char * unknown_string = \"?\";\n";
 print OUTPUT "\n";
 print OUTPUT "void detect_language_and_encoding(const char * data,LanguageAndEncodingResult * retBuffer,int iFlags = 0)\n";
@@ -878,7 +863,8 @@ print OUTPUT "	else retBuffer->dAccuracy = 0.0;\n";
 print OUTPUT "}\n";
 print OUTPUT "\n";
 
-
+print OUTPUT "/*\n";
+print OUTPUT " * this file can be compiled also as a standalone app for testing\n";
 print OUTPUT "int main(int argc,char ** argv)\n";
 print OUTPUT "{\n";
 print OUTPUT "	FILE * f = fopen(argv[1],\"r\");\n";
@@ -894,7 +880,8 @@ print OUTPUT "		printf(\"LANGUAGE %s, ENCODING %s, SCORE: %f\\n\",r.match[i].szL
 print OUTPUT "	printf(\"Accuracy: %f\\n\",r.dAccuracy);\n";
 print OUTPUT "	return 0;\n";
 print OUTPUT "}\n";
-
+print OUTPUT "*/\n";
+print OUTPUT "\n";
 
 close(OUTPUT);
 
diff --git a/src/modules/language/langdetector.cpp b/src/modules/language/langdetector.cpp
new file mode 100644
index 000000000..979e840f1
--- /dev/null
+++ b/src/modules/language/langdetector.cpp
@@ -0,0 +1,46 @@
+//
+//   File : helpwindow.cpp
+//   Creation date : Tue Aug 11 2000 18:08:22 by Szymon Stefanek
+//
+//   This file is part of the KVirc irc client distribution
+//   Copyright (C) 1999-2000 Szymon Stefanek (pragma at kvirc dot net)
+//
+//   This program is FREE software. You can redistribute it and/or
+//   modify it under the m_terms of the GNU General Public License
+//   as published by the Free Software Foundation; either version 2
+//   of the License, or (at your opinion) any later version.
+//
+//   This program is distributed in the HOPE that it will be USEFUL,
+//   but WITHOUT ANY WARRANTY; without even the implied warranty of
+//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//   See the GNU General Public License for more details.
+//
+//   You should have received a copy of the GNU General Public License
+//   along with this program. If not, write to the Free Software Foundation,
+//   Inc. ,59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+//
+
+#include <QFile>
+
+#include "langdetector.h"
+
+KviLangDetetector::KviLangDetetector(QString doc)
+{
+	qint64 iMaxSize = 4096;
+	char buffer[4096];
+	QFile *file = new QFile(doc);
+
+	if (!file->open(QIODevice::ReadOnly | QIODevice::Text))
+		return;
+
+	if(file->read(buffer, iMaxSize)<1)
+		return;
+
+	LanguageAndEncodingResult r;
+	detect_language_and_encoding(buffer,&r,0);
+	for(int i=0;i<DLE_NUM_BEST_MATCHES;i++)
+		printf("LANGUAGE %s, ENCODING %s, SCORE: %f\n",r.match[i].szLanguage,r.match[i].szEncoding,r.match[i].dScore);
+	printf("Accuracy: %f\n",r.dAccuracy);
+	return;
+}
+
diff --git a/src/modules/language/langdetector.h b/src/modules/language/langdetector.h
new file mode 100644
index 000000000..e9beeae28
--- /dev/null
+++ b/src/modules/language/langdetector.h
@@ -0,0 +1,37 @@
+#ifndef _LANGDETECTOR_H_
+#define _LANGDETECTOR_H_
+//
+//   File : langdetector.h
+//   Creation date : Sun May 11 2008 00:07:20 CET by Szymon Stefanek
+//
+//   This file is part of the KVirc irc client distribution
+//   Copyright (C) 1999-2008 Szymon Stefanek (pragma at kvirc dot net)
+//
+//   This program is FREE software. You can redistribute it and/or
+//   modify it under the terms of the GNU General Public License
+//   as published by the Free Software Foundation; either version 2
+//   of the License, or (at your opinion) any later version.
+//
+//   This program is distributed in the HOPE that it will be USEFUL,
+//   but WITHOUT ANY WARRANTY; without even the implied warranty of
+//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//   See the GNU General Public License for more details.
+//
+//   You should have received a copy of the GNU General Public License
+//   along with this program. If not, write to the Free Software Foundation,
+//   Inc. ,59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+//
+
+#include "kvi_string.h"
+#include "kvi_kvs_array.h"
+
+#include "detector.h"
+
+class KviLangDetetector
+{
+public:
+	KviLangDetetector(QString doc);
+	~KviLangDetetector();
+};
+
+#endif //_LANGDETECTOR_H_
diff --git a/src/modules/language/libkvilanguage.cpp b/src/modules/language/libkvilanguage.cpp
index c7ce80a9c..c2fe14a64 100644
--- a/src/modules/language/libkvilanguage.cpp
+++ b/src/modules/language/libkvilanguage.cpp
@@ -24,19 +24,64 @@
 
 #include "kvi_module.h"
 #include "kvi_app.h"
+#include "kvi_locale.h"
 
+#include <QFileInfo>
+
+#include "langdetector.h"
+
+/*
+	@doc: language.detect
+	@type:
+		command
+	@title:
+		language.detect
+	@short:
+		(Tries to) detect the language and encoding of a text file
+	@syntax:
+		language.detect [document: string]
+	@description:
+		Tries to detect the language and encoding of the file specified
+		as [document]. The file has to be preferably a simple text file,
+		or the detector can be deceived by the file format (eg. html files
+		contains english words inside html tags).
+		[document] can be an absolute path.
+		This command is exported by the "language" module.
+*/
+
+
+static bool language_kvs_cmd_detect(KviKvsModuleCommandCall * c)
+{
+	QString doc;
+
+	KVSM_PARAMETERS_BEGIN(c)
+		KVSM_PARAMETER("document",KVS_PT_STRING,KVS_PF_OPTIONAL,doc)
+	KVSM_PARAMETERS_END(c)
+
+	if(doc.isEmpty()){
+		c->warning(__tr2qs("No file given, no detection occured."));
+		return true;
+	}
+
+	QFileInfo * f= new QFileInfo(doc);
+	qDebug("Path %s",doc.toUtf8().data());
+	if(f)
+	{
+		if(!f->exists())
+		{
+			c->warning(__tr2qs("The file you specified doesn't exists, can't detect its language."));
+			return true;
+		}
+	}
+
+	KviLangDetetector *w = new KviLangDetetector(doc);
+
+	return true;
+}
 
 static bool language_module_init(KviModule * m)
 {
-	/*
-	m->registerFunction("nick",my_module_fnc_nick);
-	m->registerFunction("user",my_module_fnc_user);
-	m->registerFunction("host",my_module_fnc_host);
-	m->registerFunction("ip",my_module_fnc_ip);
-	m->registerFunction("server",my_module_fnc_server);
-	m->registerFunction("network",my_module_fnc_network);
-	m->registerFunction("umode",my_module_fnc_umode);
-	*/
+	KVSM_REGISTER_SIMPLE_COMMAND(m,"detect",language_kvs_cmd_detect);
 	return true;
 }